feat(tui): wire simulation control server

packages/opencode/specs/simulation-phases.md

@@ -0,0 +1,165 @@
+# Simulation Implementation Phases
+
+Status: implementation plan for `specs/simulation.md`.
+
+The full simulation architecture is intentionally broad. This document breaks it into phases that can be implemented and reviewed incrementally.
+
+## Phase 1: Control Surface And Observability
+
+Goal: start the normal app in simulation mode and inspect/drive the TUI through an external WebSocket driver.
+
+This phase proves the core shape without swapping every foundational layer yet.
+
+Implementation checklist:
+
+- [x] Add `OPENCODE_SIMULATION=1` activation in V1/full-TUI startup.
+- [x] Add simulation trace service with in-memory append-only records.
+- [x] Add OpenTUI UI state extraction for screen, focus, elements, and generated actions.
+- [x] Add OpenTUI UI action execution for typing, keys, enter, arrows, focus, and click.
+- [x] Add reusable JSON-RPC WebSocket server on `127.0.0.1:40900+`.
+- [x] Expose `ui.state`, `ui.action`, `ui.render`.
+- [x] Expose `trace.list`, `trace.clear`, `trace.export`.
+- [x] Wire visible V1/full-TUI renderer path through the same action protocol.
+- [ ] Verify a local driver can inspect state and execute a real TUI input.
+
+Scope:
+
+- Add `OPENCODE_SIMULATION=1` activation.
+- Start a TUI-owned JSON-RPC WebSocket server on `127.0.0.1:40900+`.
+- Expose `ui.state`, `ui.action`, `ui.render`.
+- Use the old simulation action model: type text, press keys, press enter, arrows, focus, click.
+- Support fake OpenTUI renderer and visible renderer through the same action protocol.
+- Add in-memory append-only trace with `trace.list`, `trace.clear`, `trace.export`.
+- Record UI observations, generated actions, executed actions, errors, and render/stabilization events.
+
+Done when:
+
+- `OPENCODE_SIMULATION=1 bun run dev` starts the normal app.
+- A local driver can connect to the WebSocket.
+- The driver can inspect current screen/elements/actions.
+- The driver can execute real TUI inputs.
+- The trace shows observations and actions.
+
+Out of scope:
+
+- Backend layer replacement.
+- Model-based runner.
+- Generated plugin config.
+- Deterministic replay tests.
+
+## Phase 2: Foundational Simulation Layers
+
+Goal: make the app safe and controlled by swapping the lowest layers, not app logic.
+
+Scope:
+
+- Wire simulation replacements through `AppNodeBuilder.build(...)` and `AppNodeBuilderV1.build(...)`.
+- Add temp-directory-backed filesystem isolation.
+- Route config/data/state/cache/temp paths into the simulation temp root.
+- Deny host filesystem escapes loudly.
+- Add simulated network registry and deny unknown external network by default.
+- Add scriptable LLM boundary.
+- Add simulated process registry:
+  - shell through `just-bash` against the simulated filesystem.
+  - minimal fake `git` support for discovery/status paths.
+  - deny unsupported process spawns.
+- Add simulation-gated backend control routes, proxied only through the frontend WebSocket.
+- Expose backend methods through the frontend server: filesystem seed/write, network register, LLM enqueue, backend snapshot.
+- Trace filesystem, network, LLM, process, and backend control activity.
+
+Done when:
+
+- Unknown network fails with a simulation error.
+- Host filesystem escape fails with a simulation error.
+- A driver can seed a project filesystem.
+- A driver can enqueue an LLM script and submit a prompt through the TUI.
+- The real session/tool path consumes the scripted LLM behavior.
+- Shell commands use `just-bash`; unsupported process spawns fail.
+- Trace contains backend activity and snapshots.
+
+Out of scope:
+
+- Model-based generation.
+- Generated plugin config state.
+- Shrinking.
+
+## Phase 3: Generated Config And Model-Based Runner
+
+Goal: explore different app states using generated commands and plugin-provided config state.
+
+Scope:
+
+- Add generated simulation plugins as the primary config-state generation mechanism.
+- Support generated plugin domains for:
+  - agents and defaults.
+  - provider/model availability.
+  - tool definitions and scripted tool behavior.
+  - MCP-like capabilities or endpoints.
+  - permission policies.
+  - instructions/system-context-like inputs where supported.
+  - workspace/project adapters where supported.
+- Add runner commands to generate, enable, disable, and inspect generated plugin state.
+- Build a custom external model-based runner, not `fast-check` yet.
+- Runner command shape: precondition, execute, model update, postcondition.
+- Runner model tracks only high-level observational state: screen category, prompt availability, sessions, files, queued LLM scripts, generated plugins, backend status, idle expectation.
+- Generate valid command sequences from model state and current `ui.state.actions`.
+- Record seed, command distribution, precondition rejections, generated plugin/config domain coverage, UI action coverage, and backend event coverage.
+
+Done when:
+
+- A seeded runner can generate a short valid exploration.
+- The runner can generate plugin-provided config state without generating large arbitrary config files.
+- The app loads and observes generated plugin state through normal plugin/config paths.
+- The runner can type and submit prompts through the TUI using generated actions.
+- Basic properties run after commands: no crash, no unknown network, no host FS escape, coherent stabilized state.
+- Trace export includes enough state to replay the generated run later.
+
+Out of scope:
+
+- Shrinking.
+- Coverage-guided mutation corpus.
+- Differential testing.
+- CI randomized runs.
+
+## Phase 4: Replay, Promotion, And Campaigns
+
+Goal: turn exploratory simulation into durable tests and prepare for larger campaigns.
+
+Scope:
+
+- Add replay from exported trace.
+- Add deterministic replay test generation from successful or failing traces.
+- Add stronger trace schema validation.
+- Add property families beyond no-crash:
+  - durable prompt admission is not lost.
+  - no duplicated visible message IDs.
+  - no orphan tool results.
+  - queue/steer semantics hold at stabilization boundaries.
+  - interrupt/resume does not duplicate promoted inputs.
+- Add corpus storage for interesting traces.
+- Add simple coverage/novelty scoring over UI states, backend event types, tool outcomes, generated config domains, and errors.
+- Add long-running campaign mode outside normal CI.
+
+Done when:
+
+- A trace from Phase 3 can be replayed deterministically.
+- A trace can be promoted to a normal test fixture.
+- Campaign runs can collect interesting traces without committing randomized tests to CI.
+- Failures produce a compact reproduction command and trace export.
+
+Out of scope:
+
+- Full shrinking.
+- Deterministic scheduler/clock control.
+- Parallel campaigns.
+- Differential testing across app versions.
+
+## Later Work
+
+- Shrinking failed traces.
+- Coverage-guided mutation of structured traces.
+- `fast-check` integration if the custom runner becomes too limited.
+- Differential testing across versions, renderers, storage modes, or scheduler policies.
+- Deterministic clock/random/scheduler control.
+- Parallel isolated workers.
+- Model-generated properties with validity/soundness/coverage scoring.

packages/opencode/specs/simulation-research/architecture-synthesis.md

@@ -0,0 +1,270 @@
+# Architecture Synthesis For Opencode Simulation
+
+## Core Direction
+
+The simulation system should be a real-app, TUI-driven, stateful model-based testing harness. It should not be a custom app command, a forked backend, or a broad reimplementation of opencode services.
+
+The production app should run normally with one required flag:
+
+```sh
+OPENCODE_SIMULATION=1 bun run dev
+```
+
+That flag enables narrow foundational layer replacements and starts a frontend-owned simulation WebSocket server. The backend server remains the normal server; simulation-only backend controls are gated behind the flag and are only reached through the frontend control server.
+
+## Control Surface
+
+- Frontend TUI process owns the external control server.
+- Protocol: JSON-RPC 2.0 over WebSocket.
+- Bind loopback only.
+- Start at port `40900`, scan upward if occupied, and log/display the actual URL.
+- External drivers never talk directly to backend simulation routes.
+- The frontend proxies backend simulation control over the same transport the TUI already uses.
+
+Initial JSON-RPC method families:
+
+- `ui.state`: screen, elements, focus, generated executable actions.
+- `ui.action`: execute real user-level inputs through OpenTUI APIs.
+- `ui.render`: force/wait for render and return state.
+- `backend.filesystem.seed` / `backend.filesystem.write`.
+- `backend.network.register`.
+- `backend.llm.enqueue`.
+- `backend.snapshot`.
+- `trace.list` / `trace.clear` / `trace.export`.
+- `run.stabilize`: wait for TUI/backend quiescence and return observations.
+
+## Action Model
+
+Actions must stay at real user-input level. The old branch's action vocabulary is the right starting point:
+
+```ts
+type UIAction =
+  | { type: "typeText"; text: string }
+  | { type: "pressKey"; key: string; modifiers?: KeyModifiers }
+  | { type: "pressEnter" }
+  | { type: "pressArrow"; direction: "up" | "down" | "left" | "right" }
+  | { type: "focus"; target: number }
+  | { type: "click"; target: number; x: number; y: number }
+```
+
+`ui.state` should return both:
+
+- `elements`: focusable/clickable/editor renderables.
+- `actions`: generated executable actions derived from those elements.
+
+Fake OpenTUI renderer and visible terminal renderer should both adapt to this same action protocol.
+
+## Foundational Layer Replacement
+
+Simulation should plug into current `AppNodeBuilder.build(...)` and `AppNodeBuilderV1.build(...)` replacement seams.
+
+Acceptable core boundaries to swap:
+
+- `FileSystem.FileSystem` / `FSUtil.Service` where needed.
+- `HttpClient.HttpClient`.
+- `RequestExecutor` / `LLMClient` or equivalent lowest LLM boundary.
+- Process spawner.
+- Global paths / env / temp paths.
+- Clock/random later, if needed.
+- Database path or in-memory DB when simulation requires isolation.
+
+Avoid replacing application services unless a core boundary is impossible:
+
+- Do not replace `SessionPrompt`, `SessionProcessor`, `ToolRegistry`, route trees, or app-level provider abstractions as the default approach.
+- Do not add a custom `simulate` command as the primary execution path.
+- Do not implement a parallel fake app.
+
+## Trace Model
+
+The frontend simulation server owns an in-memory append-only trace log by default.
+
+Trace records should include:
+
+- Run metadata: app version, seed, renderer mode, simulation URL.
+- Initial world setup: filesystem, config, environment, backend state.
+- UI observations: screen, elements, generated actions, focused item.
+- UI actions: command sent, concrete OpenTUI action, timestamp/order.
+- Backend events: session events, provider requests, tool calls/results, permission decisions, status transitions.
+- LLM scripts consumed and provider stream chunks.
+- Filesystem and network operations/diffs.
+- Stabilization boundaries and idle checks.
+- Property checks and failures.
+- Driver/model prompts and decisions when a model is driving exploration.
+
+The trace must support:
+
+- `trace.list`: query recent records.
+- `trace.clear`: reset in-memory log.
+- `trace.export`: produce JSON suitable for replay or test generation.
+
+File writing is not required initially, but the trace schema should be JSONL-friendly.
+
+## Model-Based Core
+
+The architecture should distinguish five concepts:
+
+```ts
+type Scenario = InitialWorld & { actions: readonly Action[] }
+type Action = UIAction | BackendControlAction | DriverAction
+type Observation = UIObservation | BackendObservation | TraceObservation
+type Model = abstract state machine over observable domains
+type Property = (model, observations) => pass | fail | inconclusive
+```
+
+The model should be an executable specification, not a copy of production implementation.
+
+Good model properties:
+
+- Abstract.
+- Observable.
+- Finite nondeterminism.
+- Explicit preconditions and postconditions.
+- Valid transition relation.
+- Allows multiple valid outcomes where opencode behavior is intentionally nondeterministic.
+
+Bad model properties:
+
+- Reimplements `SessionRunner` or provider streaming.
+- Mirrors database internals exactly.
+- Uses the same branch logic as production code.
+- Requires complete knowledge of implementation scheduling.
+
+## Quiescence
+
+Quiescence is a first-class command, not a timeout hidden in tests.
+
+`run.stabilize` should check, as far as possible:
+
+- TUI render is idle.
+- No active runner for the target session.
+- No eligible durable input remains unprocessed unless intentionally queued.
+- No running tool calls remain.
+- Session status is stable.
+- Projected transcript is stable across repeated reads.
+- Backend event stream has no immediate pending event burst.
+
+If quiescence cannot be proved, the command should return structured uncertainty instead of silently sleeping.
+
+## Properties And Oracles
+
+Initial built-in properties should be simple and high-signal:
+
+- No frontend/backend crash.
+- No unknown external network unless explicitly registered.
+- No host filesystem escape.
+- No duplicated visible message IDs.
+- No orphan tool results.
+- Durable prompt admission is not lost.
+- Exact prompt retry reconciles or fails according to identity rules.
+- Queue/steer promotion preserves documented delivery semantics.
+- Interrupt/resume does not duplicate promoted inputs.
+- Stabilized sessions have coherent status and transcript.
+
+Use multiple oracle styles:
+
+- Invariant checks after every step.
+- Model/refinement checks against allowed abstract outcomes.
+- Metamorphic checks across related runs.
+- Differential checks across app versions, renderers, or storage modes.
+
+## Generation Strategy
+
+Start deterministic and structured.
+
+Inputs to generate:
+
+- Workspace filesystem shapes.
+- Config variants.
+- Auth/provider/model states.
+- Session histories.
+- Prompt text and delivery modes.
+- LLM scripts with text, tool calls, errors, chunk boundaries.
+- Tool outcomes and permission decisions.
+- UI action sequences chosen from `ui.state.actions`.
+- Interrupt/restart/crash points.
+
+Use fast-check later for the external runner:
+
+- `asyncProperty` for E2E simulation.
+- `commands` for model-based command sequences.
+- Seed/path replay.
+- Shrinking action sequences.
+
+Do not run huge property suites in CI by default. Use simulation to discover/minimize traces, then generate normal deterministic tests for CI.
+
+## Coverage Guidance
+
+The simulation server and runner should collect effectiveness signals from day one:
+
+- UI element/action coverage.
+- App route/screen coverage.
+- Backend event type coverage.
+- Session state transition coverage.
+- Tool/permission outcome coverage.
+- Network route coverage.
+- Filesystem operation coverage.
+- Error/retry/recovery coverage.
+- Discarded/precondition-failed cases.
+
+Later, preserve interesting traces as corpus seeds and mutate them structurally.
+
+## Shrinking And Test Generation
+
+Shrinking should operate on semantic trace structures:
+
+- Remove actions.
+- Reduce prompt text.
+- Reduce filesystem size.
+- Remove unrelated files/config fields.
+- Simplify LLM scripts.
+- Remove chunks while preserving tool-call validity.
+- Move or remove interrupt/restart points.
+- Reduce UI navigation before the failing action.
+
+Generated normal tests should include:
+
+- Minimal initial world.
+- Explicit UI action sequence.
+- Explicit LLM/tool/network scripts.
+- Stabilization calls.
+- Deterministic assertions over semantic observations.
+
+## LLM/Agent Driver Feedback
+
+The same WebSocket observations should support model-driven exploration.
+
+Expose enough semantic context for a model to reason:
+
+- Screen text.
+- TUI elements with roles, labels, selectors, positions, capabilities.
+- Available actions.
+- Backend snapshot summaries.
+- Recent trace records.
+- Filesystem summary/diff.
+- Network log.
+- Session/message/tool summary.
+- Property failures and uncertainty.
+
+Generated properties from models should follow a lifecycle:
+
+- Proposed from evidence.
+- Translated to executable checks.
+- Validity checked.
+- Soundness checked on normal behavior.
+- Coverage/adversarial checked where possible.
+- Accepted, refined, or rejected with reason.
+
+## First Implementation Milestone
+
+Minimal useful milestone:
+
+1. `OPENCODE_SIMULATION=1` activates simulation replacements and frontend control server.
+2. Frontend starts JSON-RPC WebSocket on `127.0.0.1:40900+`.
+3. `ui.state`, `ui.action`, `ui.render`, `trace.list`, `trace.clear`, `trace.export` work.
+4. Fake and visible OpenTUI renderers share the same action protocol.
+5. Backend simulation controls are proxied through frontend WebSocket.
+6. Backend has gated control routes only when simulation flag is enabled.
+7. In-memory trace records UI actions, observations, backend control calls, and backend snapshots.
+8. Foundational replacements cover at least LLM and network denial/registration.
+9. A simple driver can seed LLM response, type a prompt through the TUI, press enter, stabilize, and assert no crash plus response visible.
+10. The resulting trace can be exported as deterministic JSON for later replay/test generation.

packages/opencode/specs/simulation-research/paper-analyses.md

@@ -0,0 +1,290 @@
+# Paper Analyses For Simulation Architecture
+
+This document extracts implementation-relevant lessons from the local paper corpus. The focus is opencode's proposed simulation system: real TUI-driven execution, narrow foundational layer replacement, WebSocket control, trace recording, model-based/property-based exploration, and generated deterministic tests.
+
+## 2016: Mysteries of Dropbox
+
+**Main idea:** stateful PBT can find bugs in real, black-box, nondeterministic distributed systems by generating action sequences, recording observations, and checking whether the observation trace has some valid explanation under a small model.
+
+Useful techniques:
+
+- Generate commands separately from observed effects.
+- Model hidden nondeterministic events explicitly, even when the SUT does not expose them.
+- Accept traces if there exists a sequence of hidden events that makes observations valid.
+- Maintain possible model states, not a single expected state.
+- Make quiescence explicit with a `STABILIZE` command.
+- Re-run flaky failures during shrinking when nondeterminism cannot be fully controlled.
+
+Pitfalls:
+
+- Quiescence detection can lie.
+- A model that is too permissive can explain away real bugs.
+- Timing-dependent failures need either scheduler control or repeated validation.
+- Raw happens-before modeling can become elegant but impractical.
+
+Implications for opencode:
+
+- Do not compare raw transcripts only. Compare observed behavior against allowed abstract outcomes.
+- Represent hidden runtime transitions like prompt promotion, session wake, provider turn continuation, tool completion, retries, interrupt delivery, and event projection.
+- Add a `drainUntilIdle` or `stabilize` simulation command with strict checks.
+- Record all generated commands, UI actions, backend events, provider scripts, tool results, and snapshots in an append-only trace.
+- Shrinking should preserve semantic validity and revalidate nondeterministic failures.
+
+## 2019: Coverage Guided, Property Based Testing
+
+**Main idea:** plain random generators often fail when valid inputs have sparse semantic preconditions. Coverage-guided PBT keeps interesting inputs and mutates structured values to explore deeper states.
+
+Useful techniques:
+
+- Maintain a corpus of inputs that improve coverage.
+- Mutate typed structures rather than raw bytes.
+- Keep both successful seeds and promising discarded seeds.
+- Fall back to random generation when mutation stalls.
+- Use coverage and progress counters, not only binary edge coverage.
+
+Pitfalls:
+
+- Instrumenting irrelevant framework code hurts performance.
+- Generic mutators can explode in search space.
+- Expert handwritten generators still outperform generic mutation but are expensive.
+
+Implications for opencode:
+
+- Preserve interesting `Scenario` and `Trace` seeds.
+- Add structured mutators for prompts, tool calls, provider chunks, permission decisions, filesystems, config, interrupt timing, crash/restart points, and scheduler actions.
+- Track semantic novelty: event types, session states, tool outcomes, permission branches, replay/recovery paths, and UI routes.
+- Avoid byte fuzzing as the core; use it only inside fields that are naturally bytes/text.
+
+## 2021: Model-Based Testing In Practice
+
+**Main idea:** MBT works in industrial E2E systems when models are pragmatic, visible, and integrated into normal automation. Graph-like models are useful because actions and assertions are explicit and coverage is understandable.
+
+Useful techniques:
+
+- Model nodes as states/checkpoints and edges as actions.
+- Split large systems into small composable models.
+- Use traversal strategies, weights, and stop conditions.
+- Report paths, coverage, and model transitions.
+
+Pitfalls:
+
+- Heavy formal models reduce adoption.
+- Auto-inferred models can be noisy and costly to clean up.
+- Coverage metrics must be live and inspectable.
+
+Implications for opencode:
+
+- Start with a small simulation DSL, not a complete formal model.
+- Model domains separately: session lifecycle, prompt admission, queue/steer, tool execution, permissions, interrupts, compaction, crash/restart.
+- Track command coverage, transition coverage, property coverage, and failure-mode coverage.
+- Keep generated failure output readable: model path, user inputs, app observations, violated invariant.
+
+## 2022: Property-Based Testing For Metamorphic Testing
+
+**Main idea:** metamorphic testing helps when exact expected outputs are unavailable. It checks relations between multiple executions or transformed inputs.
+
+Useful techniques:
+
+- Generate source cases, derive follow-up cases, and compare related outputs.
+- Use generators and shrinkers that preserve relation validity.
+- Combine multiple metamorphic relations.
+
+Pitfalls:
+
+- Weak metamorphic relations miss real faults.
+- Naive shrinkers can break validity.
+- Reimplementing production logic in the oracle makes the test useless.
+
+Implications for opencode:
+
+- Use metamorphic relations for nondeterministic model behavior.
+- Examples:
+  - Same prompt ID and same delivery mode should reconcile exactly on retry.
+  - Queueing independent prompts should preserve durable admission order.
+  - Interrupt/resume should not duplicate promoted user messages or orphan tool results.
+  - Crash after durable admission should not invent provider work unless recovery explicitly permits it.
+  - Fake renderer and visible renderer should agree on semantic action results.
+  - Two app versions should satisfy the same semantic invariants for the same trace.
+
+## 2022: Climbing The Stairway To Verification
+
+**Main idea:** PBT becomes stronger when it mirrors a refinement/specification structure. The test asks whether implementation behavior refines an executable abstract model.
+
+Useful techniques:
+
+- Generate one canonical test case and project it into abstract and concrete worlds.
+- Compare implementation output to a finite set of model-allowed outcomes.
+- Keep models abstract and observable.
+- Use executable specs as cheaper, incremental versions of formal proofs.
+
+Pitfalls:
+
+- The model can become a second implementation.
+- Strong preconditions plus random generation cause excessive discarded tests.
+- Overly abstract nondeterminism can explode.
+
+Implications for opencode:
+
+- Build simulation around `scenario -> model outcomes -> real app run -> relation check`.
+- The model should represent visible session, message, tool, provider, permission, event, status, and filesystem effects.
+- The model must not reimplement `SessionRunner`, provider streaming, tool registry, or Effect scheduling.
+- Generate concrete scenarios and derive abstract model inputs from them.
+
+## 2023: QuickerCheck
+
+**Main idea:** PBT and shrinking can be parallelized, especially for expensive properties, if workers have isolated state and reproducible seeds.
+
+Useful techniques:
+
+- Give each worker its own PRNG seed and size schedule.
+- Stop all workers after the first counterexample.
+- Run cleanup/finalizers for interrupted effectful properties.
+- Use greedy parallel shrinking when deterministic minimality is less important than speed.
+
+Pitfalls:
+
+- Shared filesystem/global state breaks parallel PBT.
+- Cancellation can leave processes, files, sockets, or locks behind.
+- Parallel shrinking can be slower for cheap properties.
+
+Implications for opencode:
+
+- Design simulation workers as isolated from the start: workspace, DB, ports, fake providers, random seeds, trace buffers.
+- Use `(campaignSeed, workerID, caseIndex)` for reproducibility.
+- Separate fast local runs from long parallel campaigns.
+- Add cleanup boundaries for every case.
+
+## 2024: Can Large Language Models Write Good Property-Based Tests?
+
+**Main idea:** LLMs can synthesize useful PBTs, but generated tests must be validated for validity, soundness, and property coverage. Two-stage prompting outperforms monolithic generation.
+
+Useful techniques:
+
+- First extract properties, then generate tests for one property at a time.
+- Classify failures as invalid test, unsound property, weak property, or real bug.
+- Use mutation/property coverage to check whether a property actually detects violations.
+
+Pitfalls:
+
+- Passing generated tests can be weak.
+- LLMs overgeneralize documentation and miss implicit preconditions.
+- Mutation coverage can be noisy if mutants are invalid or equivalent.
+
+Implications for opencode:
+
+- Treat model-generated simulation properties as candidates.
+- Store property lifecycle: proposed, executable, validity-checked, soundness-checked, coverage-scored, accepted, rejected.
+- Generate small focused properties, not one huge “test opencode” property.
+- Expose enough observations for a model to validate its own property assumptions.
+
+## 2024: Property-Based Testing In Practice
+
+**Main idea:** experienced developers use PBT in a small number of high-leverage patterns. The hardest parts are writing useful properties, writing generators, shrinking, and knowing whether passing tests mean anything.
+
+Useful techniques:
+
+- High-leverage patterns include differential testing, model-based tests, round trips, catastrophic failure properties, and invariants.
+- Developers validate PBT effectiveness through mutation testing, example inspection, code coverage, property coverage, and supplementary example tests.
+- Passing tests need inspectable generated examples and distribution feedback.
+
+Pitfalls:
+
+- Derived generators can create false confidence.
+- Shrinkers can violate invariants.
+- Slow PBTs get removed.
+
+Implications for opencode:
+
+- Provide built-in property families instead of requiring every contributor to invent properties.
+- Show generator stats: action distribution, trace length, discarded cases, transition coverage, example traces.
+- Make failure output a concise, reviewable artifact.
+- Support “promote minimized trace to normal test.”
+
+## 2026: Agentic PBT
+
+**Main idea:** an agentic loop can generate better PBTs than one-shot prompting by inspecting code/docs, proposing evidence-backed properties, running tests, triaging failures, refining false alarms, and reporting only reproducible bugs.
+
+Useful techniques:
+
+- Use a structured loop: inspect, propose, execute, triage, refine, report.
+- Prefer high-value property patterns: invariants, round trips, inverse operations, multiple implementations, laws, confluence, metamorphic relations, and no-crash parser entrypoints.
+- Keep an evidence chain for every property.
+
+Pitfalls:
+
+- Intent ambiguity is the main false-positive source.
+- Internal helpers often have implicit preconditions.
+- Extreme generated inputs can be unrealistic.
+
+Implications for opencode:
+
+- The simulation system should be friendly to model-driven exploration, not just batch tests.
+- Store prompts, observations, selected actions, available actions, traces, refinements, and final classifications.
+- Add a triage workflow before surfacing model-generated failures as bugs.
+
+## 2026: Evolution Of Python Tests Into PBT
+
+**Main idea:** existing example and parameterized tests often evolve naturally into PBTs. Generated deterministic tests and PBTs should be connected, not treated as separate worlds.
+
+Useful techniques:
+
+- Convert constants/parameter tables into generators.
+- Keep explicit examples for known edge cases.
+- Adjust generators/settings over time as tests mature.
+
+Pitfalls:
+
+- Coverage can be inflated by harness/generator code.
+- PBTs can fail early and cover fewer later assertions.
+- Slow PBTs are removed.
+
+Implications for opencode:
+
+- Use existing tests as simulation corpus seeds.
+- Convert minimized simulation traces into normal deterministic tests.
+- Keep SUT coverage separate from simulation harness coverage.
+- Store generated regressions as explicit fixtures.
+
+## 2026: Natural Language To Executable Properties For Mobile Apps
+
+**Main idea:** natural-language app properties can become executable UI properties if the system first performs semantic grounding of UI elements.
+
+Useful techniques:
+
+- Decompose property synthesis into UI semantic grounding and executable property synthesis.
+- Represent properties as precondition, interaction scenario, postcondition.
+- Enrich each widget with text, ID, type, semantic label, functionality, screenshot/crop, and provenance.
+
+Pitfalls:
+
+- Similar widgets cause grounding errors.
+- Free-form natural language is less reliable than structured Given/When/Then descriptions.
+- Incorrect preconditions/postconditions are more common than incorrect interactions.
+
+Implications for opencode:
+
+- Expose semantic TUI state, not just screen text and coordinates.
+- Elements/actions should have stable IDs, roles, labels, capabilities, focus/click/edit metadata, visibility, and provenance.
+- Generated properties should use a precondition/action/postcondition structure.
+
+## 2026: PropGen Mobile App Testing
+
+**Main idea:** properties can be generated from runtime behavioral evidence. The loop is exploration, evidence collection, property synthesis, executable translation, testing, feedback, and refinement.
+
+Useful techniques:
+
+- Record condition-action-outcome traces.
+- Use functionality-guided exploration plus random exploration fallback.
+- Refine imprecise properties by classifying whether the problem is precondition, interaction, or postcondition.
+
+Pitfalls:
+
+- Single traces cause overfitting.
+- Generated properties can assert incidental UI details.
+- Refinement can overfit unless anchored to original evidence.
+
+Implications for opencode:
+
+- Record rich traces with before-state, action intent, concrete action, model/provider/tool effects, after-state, state diff, and outcome label.
+- Let models derive properties from observed behavior, then validate and refine them.
+- Use both model-guided goals and stochastic action exploration.

packages/opencode/specs/simulation-research/papers/.gitattributes

@@ -0,0 +1 @@
+*.pdf binary

packages/opencode/specs/simulation-research/sources.md

@@ -0,0 +1,24 @@
+# Simulation Research Sources
+
+Local research corpus for the opencode simulation architecture work.
+
+## Downloaded Papers
+
+| Year | Paper | Local PDF | Extracted Text | Source |
+| --- | --- | --- | --- | --- |
+| 2016 | Mysteries of Dropbox: Property-Based Testing of a Distributed Synchronization Service | `papers/2016-mysteries-of-dropbox.pdf` | `text/2016-mysteries-of-dropbox.txt` | https://publications.lib.chalmers.se/records/fulltext/232551/local_232551.pdf |
+| 2019 | Coverage Guided, Property Based Testing | `papers/2019-coverage-guided-pbt.pdf` | `text/2019-coverage-guided-pbt.txt` | https://lemonidas.github.io/pdf/FuzzChick.pdf |
+| 2021 | Model-Based Testing in Practice: An Experience Report from the Web Applications Domain | `papers/2021-mbt-web-applications-practice.pdf` | `text/2021-mbt-web-applications-practice.txt` | https://arxiv.org/pdf/2104.02152 |
+| 2022 | Application of Property-Based Testing Tools for Metamorphic Testing | `papers/2022-pbt-for-metamorphic-testing.pdf` | `text/2022-pbt-for-metamorphic-testing.txt` | https://arxiv.org/pdf/2211.12003 |
+| 2022 | Property-Based Testing: Climbing the Stairway to Verification | `papers/2022-stairway-to-verification.pdf` | `text/2022-stairway-to-verification.txt` | https://trustworthy.systems/publications/papers/Chen_ROSKHK_22.pdf |
+| 2023 | QuickerCheck: Implementing and Evaluating a Parallel Run-Time for QuickCheck | `papers/2023-quickercheck.pdf` | `text/2023-quickercheck.txt` | https://arxiv.org/pdf/2404.16062 |
+| 2024 | Can Large Language Models Write Good Property-Based Tests? | `papers/2024-llms-write-good-pbt.pdf` | `text/2024-llms-write-good-pbt.txt` | https://arxiv.org/pdf/2307.04346 |
+| 2024 | Property-Based Testing in Practice | `papers/2024-pbt-in-practice.pdf` | `text/2024-pbt-in-practice.txt` | https://www.cis.upenn.edu/~bcpierce/papers/icse24-pbt-in-practice |
+| 2026 | Finding bugs across the Python ecosystem with Claude and property-based testing / Agentic PBT | `papers/2026-agentic-pbt.pdf` | `text/2026-agentic-pbt.txt` | https://arxiv.org/pdf/2510.09907 |
+| 2026 | On the Evolution of Python Test Cases into Property-based Tests | `papers/2026-evolution-python-tests-to-pbt.pdf` | `text/2026-evolution-python-tests-to-pbt.txt` | https://soft.vub.ac.be/Publications/2026/vub-tr-soft-26-02.pdf |
+| 2026 | From Natural Language to Executable Properties for Property-based Testing of Mobile Apps | `papers/2026-nl-to-executable-properties-mobile.pdf` | `text/2026-nl-to-executable-properties-mobile.txt` | https://arxiv.org/pdf/2603.21263 |
+| 2026 | From Exploration to Specification: LLM-Based Property Generation for Mobile App Testing | `papers/2026-propgen-mobile-app-testing.pdf` | `text/2026-propgen-mobile-app-testing.txt` | https://arxiv.org/pdf/2604.13463 |
+
+## Notes
+
+The ACM PDF endpoint for `Property-Based Testing in Practice` returned `403`; the author-hosted University of Pennsylvania copy was downloaded successfully.

packages/opencode/specs/simulation-research/text/2016-mysteries-of-dropbox.txt

@@ -0,0 +1,714 @@
+                                        Mysteries of Dropbox
+        Property-Based Testing of a Distributed Synchronization Service
+                             John Hughes∗† , Benjamin C. Pierce‡ , Thomas Arts∗ , Ulf Norell∗† ,
+                                                    ∗ Quviq AB, Göteborg, Sweden
+            † Dept of Computer Science and Engineering, Chalmers University of Technology, Göteborg, Sweden
+                       ‡ Dept of Computer and Information Science, University of Pennsylvania, PA, USA
+
+
+
+
+   Abstract—File synchronization services such as Dropbox are              Our goal in this paper is to present a testable formal specifi-
+used by hundreds of millions of people to replicate vital data.         cation for the core behavior of a file synchronizer. We do so via
+Yet rigorous models of their behavior are lacking. We present           a model developed using Quviq QuickCheck [4]. Despite the
+the first formal—and testable—model of the core behavior of a
+modern file synchronizer, and we use it to discover surprising          apparent simplicity of the problem, we encountered interesting
+behavior in two widely deployed synchronizers. Our model is             challenges regarding both specification and testing. We used
+based on a technique for testing nondeterministic systems that          the model to test Dropbox, Google Drive, and ownCloud (an
+avoids requiring that the system’s internal choices be made visible     open source alternative), exposing unexpected behavior in two
+to the testing framework.                                               out of three.
+                                                                           In Section II, we introduce our testing framework. Sec-
+                       I. I NTRODUCTION                                 tion III gives a high-level overview of the concepts used in our
+                                                                        model, in particular the operations performed by test cases, the
+   File synchronization services—distributed systems that
+                                                                        observations made when a test case is run on the system under
+maintain consistency among multiple copies of a file or
+                                                                        test (SUT), and the explanations that we construct to determine
+directory structure—are now ubiquitous. Dropbox claim 400
+                                                                        whether the test has passed or failed. Section IV presents the
+million users,1 while Google Drive and Microsoft OneDrive
+                                                                        formal specification itself, beginning with a naive version and
+are reported to have over 240 million users each.2 In addition
+                                                                        refining it in light of failed tests that reveal subtleties in the
+to these large-scale commercial offerings and many smaller
+                                                                        synchronizer’s handling of corner cases. Section V describes
+ones, there are a plethora of open source synchronizers,
+                                                                        further failed tests that, rather than pinpointing inadequacies
+enabling users to create their own ‘cloud storage.’ With so
+                                                                        in the specification, seem to us to exemplify unintended
+many people trusting their data to synchronization services,
+                                                                        behaviors of both Dropbox and ownCloud, including situations
+their correctness should be a high priority indeed.
+                                                                        where each system can lose data. In Section VI, we discuss the
+   Surprisingly, then, it seems that only one file synchronizer
+                                                                        pragmatics of our testing framework, in particular our methods
+has been formally specified to date: Unison [1]–[3]. However,
+                                                                        for triggering timing-dependent behaviors, for observing when
+even Unison’s specification is not directly useful for testing;
+                                                                        the system has reached quiescence, and for shrinking tests to
+moreover, Unison works rather differently from contemporary
+                                                                        minimal failing cases in the presence of nondeterminism. In
+synchronization services: it synchronizes two peers at a time,
+                                                                        Section VII, we sketch an initially promising attempt to formu-
+rather than many clients with one server; synchronization is
+                                                                        late the specification in terms of Lamport’s “happens before”
+invoked explicitly by the user, rather than taking place auto-
+                                                                        relation and explain why it ultimately proved unsuccessful.
+matically in the background; and conflict resolution involves
+                                                                        Section VIII surveys related work, and Sections IX and X
+user interaction, rather than being performed automatically.
+                                                                        present future directions and concluding remarks.
+   Synchronizers are challenging not only to specify but also              Our main contributions are as follows: (1) We construct the
+to test. They are large-scale, nondeterministic distributed sys-        first formal model of the core behavior of modern file synchro-
+tems, with timing-dependent behavior. They must cope with               nizers. (2) To define the model, we develop a technique for
+conflicts (when the same file is modified concurrently on two           testing nondeterministic systems that does not require that the
+or more clients). They work in the background, and their                system’s internal choices be visible to the testing framework.
+state is unobservable. Moreover, they are slow. They share              Our technique is based on an explicit representation of hidden
+these difficulties with a large class of critical systems; thus,        system state plus “conjectured events” that mutate it. (3) We
+techniques for addressing these problems are valuable, not              validate our final model against two commercial synchronizers
+only for testing file synchronizers, but in a broader context.          and one open-source one, showing that their behavior agrees
+                                                                        with the model in most situations. (4) We demonstrate the
+  1 techcrunch.com, “Dropbox now has more than 400 million registered
+                                                                        effectiveness of our model and testing framework by using it
+users”, June 24th , 2015.
+  2 fortune.com, “Who’s winning the consumer cloud storage wars?”       to reveal a number of surprising behaviors, likely not intended
+November 6th , 2014.                                                    by the developers of these systems.
+        II. T ESTING A S YNCHRONIZATION S ERVICE                    ones we have tested so far. In particular, we did not want to
+                                                                    assume any direct access to remote servers outside our control.
+   We begin by making a rather drastic simplifying assump-          Therefore, we treat the synchronizer as a ‘black box’, which
+tion: we consider only one file, with operations to read,           we communicate with only through the file system; our tests
+write, and delete it. We consider only operations that read         just read and write files on the virtual machines and check the
+and write the entire file; we do not specify or test how the        results for validity. This allows us to write the specification
+synchronizer interacts with open files as they are modified.        without dependencies on synchronizer-specific APIs.3
+Restricting our attention to one file may seem extreme, but
+even this simple setting forces us to confront essential issues                 III. OVERVIEW OF THE S PECIFICATION
+of synchronization—in particular subtleties arising in the pres-       Our strategy for test case generation is very simple: we
+ence of conflicts—and it is enough to expose surprising be-         generate test cases consisting of random sequences of calls to
+haviors in real systems. A plan for extending the specification     a small set of basic filesystem operations. The basic operations
+to multiple files and directories is sketched in Section IX.        we consider are
+   We developed our model using Quviq QuickCheck [4],                  • R EAD N , which reads the (one) file on node N (one of
+a descendant of Haskell QuickCheck [5]. QuickCheck tests                  the VMs), and
+properties—universally quantified boolean formulæ—by gen-              • W RITE N V , which writes the value V (a string) to the
+erating random values for the quantified variables and check-             file on node N .
+ing that the formula evaluates to true. When a test fails,          We will introduce a few additional operations below.
+QuickCheck “shrinks” it, searching for a similar but smaller           We use QuickCheck’s state machine library to generate
+test that also fails and ultimately reporting a failing test that   tests, with a trivial model state—our tests are simply random
+is ‘minimal’ in some sense. (This process is akin to delta-         sequences of R EADs and W RITEs, with random arguments.
+debugging [6], although the details are slightly different.)        One might expect to track the file contents in the model state,
+QuickCheck provides a domain-specific language for defining         and check that R EAD returns the modeled contents—but this
+test data generators and shrinking searches; using this DSL,        could only work if synchronization were instantaneous, which
+users can exert fine control over the random distribution of        of course it is not. Instead of trying to express our specification
+test cases. A part of this DSL is a notation for specifying         in this synchronous style, we collect observed events as each
+state machine models, which generate test cases consisting of       operation is actually executed, and we use a separate state
+sequences of calls to an API under test [7], [8]; this is how       machine to validate the resulting sequence of events.
+we generated test cases for synchronizers. Quviq QuickCheck            The observed events corresponding to the READ and WRITE
+is embedded in the Erlang programming language—that is,             operations are as follows:
+specifications are just Erlang programs that call libraries
+                                                                       • when R EAD N returns the value V , we observe the event
+supplied by QuickCheck—and the generated tests invoke the
+                                                                          R EADN → V , and
+SUT directly via Erlang function calls. As a result, there is
+                                                                       • when W RITE N V overwrites the value Vold , we observe
+no distinction between ‘abstract’ and ‘concrete’ test cases, as
+                                                                          the event W RITEN V → Vold .
+there often is in model-based testing, and there is no need for
+a ‘system adapter’; instead, generated test cases are directly      Notice that, when we write the file, we observe the previous
+executable.                                                         contents as well as the new one (taking the ‘previous contents’
+   We ran our tests on laptops running a host operating system      of a newly created file to be the special value ⊥). The value
+(Windows 8.1 or Mac OS) together with several virtual ma-           that we overwrite matters, because of the way synchronizers
+chines running Ubuntu Linux. The file synchronizer under test       handle conflicts. (This observation is not actually atomic, but
+was installed on each virtual machine and under the host op-        it is very unlikely that the dropbox dæmon will overwrite the
+erating system, so we could read and write files on any of the      file between our read and write, and we have not observed it
+virtual machines, or on the host, and expect the synchronizer       to happen.)
+to propagate changes to all of the others. We ran distributed          A conflict occurs when two or more clients write the
+Erlang on the virtual machines and the host, using the host         file concurrently—that is, without having seen each other’s
+to coordinate each test by making remote procedure calls            updates. For example, if client 1 writes ‘a’ to the file, and
+on the virtual machines. All machines were also connected           client 2 then writes ‘b’ before ‘a’ has been delivered to client 2
+to the internet, allowing us to test synchronization servers        by the synchronizer, a conflict is created. One of the two values
+running either remotely (in the case of Dropbox and Google          wins, and eventually all clients will see this value in the file if
+Drive), or on another VM (in the case of ownCloud). The use         there are no more writes, but conflicting values are also saved
+of multiple VMs on a single physical machine was purely             in special files in the same directory, with names derived from
+a matter of convenience: Erlang’s support for transparent             3 For testing the specification, we found it convenient to use limited
+distribution would make it straightforward to run the same          communication with the local dæmon, where available; for example, the
+setup on multiple nodes. We used 3 VMs for most tests.              Dropbox client on Ubuntu provides a handy Python script for querying
+                                                                    whether the local dæmon thinks it is up to date, which we used to speed
+   We wanted a model that would apply (perhaps with minor           up testing a bit. This adds <10 lines of code per synchronizer to the test
+adjustments) to many file synchronizers, not just the specific      harness.
+the name of the original file, such as ‘paper.tex (John’s            tests may well result in a stable state never being reached!
+conflicted copy)’; these files are eventually replicated             In practice we wait up to 30 seconds, long enough to allow
+to other clients just like any other file. To avoid depending        synchronization to finish if it is going to but short enough to
+on implementation-chosen names, our model specifies just the         enable us to ‘shrink’ test cases (which involves running many,
+set of values expected to appear in conflict files, and our test     many failing tests—see Section VI) in a reasonable amount of
+harness assumes that any files that appear in the same directory     time. After 30 seconds we record a “failed stabilization” ob-
+as the main one are conflict files.                                  servation of the form S TABILIZE → {(V1 , C1 ), . . . , (Vn , Cn )},
+   One subtlety in the specification of conflicts is that, because   where each (Vi , Ci ) records the the file contents and conflict
+conflicts can only be detected using global information, there       set on one of the nodes. Such an observation is regarded as
+may be a delay in the creation of conflict files. Consider the       invalid by the specification, so any test that generates it is
+following sequence of observations.                                  considered to fail.
+                                                                        Since stabilization is slow—and most interesting after sev-
+           Client 1               Client 2
+                                                                     eral read and write operations—we include it only 1/10 as
+           W RITE1 ‘a’ → ⊥
+                                                                     often as R EADs and W RITEs. We also add a S TABILIZE at
+                                   W RITE2 ‘b’ → ⊥
+                                                                     the end of every test case, which improves the probability of
+                                   R EAD2 → ‘a’
+                                                                     detecting that something went wrong and also reduces the risk
+The two writes are in conflict, and the write of ‘a’ has ’won’,      of one test influencing the outcome of the next one.
+so the value ‘b’ should appear in a conflict file. However,             How can we decide whether a sequence of observed events
+client 2 cannot determine this locally (since a different client     is valid? We use a separate state machine, which accepts or re-
+could have overwritten ‘b’ with ‘a’ in the meantime). So we          jects a sequence of observed events. However, the observations
+must wait for pending communications with the server to finish       we make do not tell the whole story: in the background, the
+before checking for the existence of conflict files.                 synchronizer is also performing actions. This makes our tests
+   We therefore add another operation to our test cases,             nondeterministic—we cannot tell, from the events we have
+S TABILIZE, which waits for synchronization to complete on           observed, what state the whole system is in. We address this by
+all client nodes. At this point, the same value V should be          adding conjectured events to the observed ones, representing
+in the file on all clients, and all clients should have the same     actions taken by the synchronization service (messages be-
+set of conflict files. Once the system is stable, we observe the     tween local dæmons and a central server, interactions between
+event S TABILIZE → (V, C), where C is the set of values in the       dæmons and local filesystems, etc.). In general, we can add
+conflict files (i.e., both the value and the conflict files should   conjectured events to a given sequence of observed events in
+be eventually consistent; we will see later that our model can       many different ways, each resulting in a different combined
+always reach such a state). Note that this is a system wide          sequence of observed and conjectured events, which we call an
+event, not an event observed on just one client. To check that       explanation. If any of the explanations is accepted by our state
+the correct conflict file is created in the example above, we        machine, we consider the test to have passed. If there is no
+would add a S TABILIZE operation to the end of the test, which       way to insert conjectured events so the resulting explanation
+should result in the following observed events:                      is accepted, then we consider that the test has failed.
+           Client 1               Client 2                              Our conjectured events are uploads to, and downloads from,
+           W RITE1 ‘a’ → ⊥                                           the server. We write these events as U PN and D OWNN ,
+                               W RITE2 ‘b’ → ⊥                       where N is the node taking part in the up- or down-load.
+                               R EAD2 → ‘a’                          Thus, when we model the state of the whole system, we will
+                  S TABILIZE → (‘a’, {‘b’})                          need to include the server’s state in the model. (Of course,
+                                                                     in reality ’the Dropbox server’ may itself be a replicated
+If ‘b’ were missing from the conflict set in the last observed       service involving many hosts. Our model implicitly assumes
+event, this would represent lost data and the test would fail.       strong consistency among these hosts; weaker consistency in
+   But how can we implement S TABILIZE? That is, how can             Dropbox’s implementation might in principle cause our tests
+we tell that synchronization is complete, given that we treat        to fail, but we have not observed this.)
+the synchronizer as a black box? Our solution is a little ad
+hoc. Certainly, the value in the file and the conflict files must      To make all of the above more concrete, here is an example
+be the same on each client. Also, under Ubuntu, Dropbox              of a simple test case:
+provides a handy Python script to check the local Dropbox
+                                                                                       Client 1         Client 2
+dæmon’s status; the dæmons on each client must be reporting
+                                                                                       W RITE1 ‘a’
+‘up to date’. We wait for these necessary conditions to become
+                                                                                                         W RITE2 ‘b’
+true—but they are not sufficient, since the server may still
+                                                                                       R EAD1
+be holding data that will be sent to the clients. So if a
+                                                                                                       W RITE2 ‘c’
+S TABILIZE → (V, C) operation ever leads to an observation
+                                                                                                S TABILIZE
+that would cause test failure, then we wait a bit longer and
+retry the operation. We cannot wait too long, because failed         Here is an observation that might arise from this test:
+          Client 1               Client 2                          ‘a’ as the previous value), so it cannot have been in conflict
+          W RITE1 ‘a’ → ⊥                                          with any of the other W RITEs.
+                                 W RITE2 ‘b’ → ‘a’
+           R EAD1 → ‘b’
+                               W RITE2 ‘c’ → ‘b’                              IV. F ORMALIZING THE S PECIFICATION
+                    S TABILIZE → (‘c’, ∅)
+And here is a valid explanation of this observation:                  Formally, we use a deterministic state machine to accept or
+                                                                   reject explanations. We define the state of the machine, the
+          Client 1               Client 2                          system state, as follows:
+          W RITE1 ‘a’ → ⊥
+          U P1                                                       • a global stable value ServerVal (i.e., the value currently
+                                 D OWN2                                held on the server)
+                                 W RITE2 ‘b’ → ‘a’                   • a global conflict set Conflicts (a set of values)
+
+                                 U P2                                • for each node N ,
+
+           D OWN1                                                          – a local value LocalVal N ,
+                                 W RITE2 ‘c’ → ‘b’                         – a local freshness, Fresh?N ∈ {FRESH, STALE}, and
+                                 U P2                                      – a local cleanliness, Clean?N ∈ {CLEAN, DIRTY}.
+           R EAD1 → ‘b’                                            Values (i.e., file contents) are just strings. (We define the result
+           D OWN1                                                  of a R EAD or W RITE before the file has ever been written to
+                   S TABILIZE → (‘c’, ∅)                           be the special value ⊥.)
+Of course, the same test may give rise to different observations      In the initial state Sinit , the stable value and all the local
+(and the same observation may be explained by many possible        values are ⊥, and all nodes are FRESH and CLEAN.
+explanations). For example, here is another observation that          There are three kinds of observed events (R EAD, W RITE,
+might arise from running the test above:                           and S TABILIZE) and two kinds of conjectured events (U P
+          Client 1               Client 2                          and D OWN). A sequence of observed events is called an
+                                                                   observation. A sequence of both kinds of events is called an
+          W RITE1 ‘a’ → ⊥
+                                                                   explanation. An explanation E explains an observation O if
+                                 W RITE2 ‘b’ → ⊥
+                                                                   deleting the conjectured events from E leaves just O.
+           R EAD1 → ‘a’
+                              W RITE2 ‘c’ → ‘b’                       For every event (of either kind), we will define a transition
+                  S TABILIZE → (‘a’, {‘c’})                        consisting of a precondition (which tells us whether the event
+                                                                   can happen in a given system state) and an effect (which
+A valid explanation of this observation is:                        defines the change to the system state after the event has
+                                                                   happened). Taken together, these preconditions and effects
+          Client 1               Client 2
+                                                                   define a partial function Next mapping a system state plus
+          W RITE1 ‘a’ → ⊥
+                                                                   an event to a new state (or failing, if the event’s precondition
+                                 W RITE2 ‘b’ → ⊥
+                                                                   is not satisfied by the state).
+           R EAD1 → ‘a’
+                                 W RITE2 ‘c’ → ‘b’                    An explanation E is valid with respect to some starting state
+           U P1                                                    S if either (1) E is the empty sequence of events, or else (2) E
+                              U P2                                 is a non-empty sequence e : E 0 (where : is ’cons’), such that
+                  S TABILIZE → (‘a’, {‘c’})                        Next(S, e) yields a new state S 0 and E 0 is valid with respect
+                                                                   to S 0 .
+  Conversely, suppose the synchronizer under test were mis-           An observation O is valid if it is explained by some
+behaving. Then the same test case might lead to an observation     explanation that is valid with respect to the initial state. There
+with no valid explanations. For example:                           are only finitely many valid explanations for each observation,
+          Client 1               Client 2                          as we explain below, so validity of observations is decidable.
+          W RITE1 ‘a’ → ⊥                                             A test is a sequence of operations. It fails if the observation
+                                 W RITE2 ‘b’ → ‘a’                 that arises by running it has no valid explanation; otherwise it
+           R EAD1 → ‘b’                                            succeeds.
+                              W RITE2 ‘c’ → ‘b’                       It remains only to define the transitions themselves. The
+                  S TABILIZE → (‘c’, {‘a’})                        read and write transitions are straightforward:
+Here, the final S TABILIZE observation shows that a conflict
+file has been created for the file value ‘a’. But ‘a’ was the            R EADN → V
+first value written to the file, and it must have been the first      Precondition: LocalVal N = V
+value uploaded to the server (because the second W RITE saw           Effect: none
+    W RITEN Vnew → Vold                                                  U PN
+   Precondition: LocalVal N = Vold                                      Precondition: Clean?N = DIRTY
+   Effect: LocalVal N ← Vnew                                            Effect: Clean?N ← CLEAN
+           Clean?N ← DIRTY                                                      if Fresh?N = FRESH then
+                                                                                   Fresh?N 0 ← STALE for all N 0 6= N
+A write event does have a precondition, because it observes                        ServerVal ← LocalVal N
+the value that is overwritten. That is, a read or write event on                else Conflicts ← Conflicts ∪ {LocalVal N }
+client node N is valid with respect to some model state if the
+value that the event observes in the filesystem agrees with the      U PN is only allowed if node N is DIRTY (written since the last
+model’s current value for that node. Observing a R EAD has no        D OWN). Its effect is either to update the server’s value from
+effect on the model state, while observing a W RITE changes          node N ’s if N is currently FRESH—i.e., if it is not in conflict
+the model’s local value for node N to the one that was written       with a W RITE on another node that has already reached the
+by the W RITE operation and marks node N as DIRTY.                   server—or otherwise to mark the local value as a conflict. In
+   The S TABILIZE → (V, C) event has no effects (it is like a        either case, node N is marked as CLEAN.
+R EAD in this respect), but it has a very strict precondition:          To decide whether a test succeeded, we construct a valid
+                                                                     explanation for the observation we made; that is, we insert
+                                                                     a sequence of U P and D OWN events between each pair of
+    S TABILIZE → (V, C)                                              observed events that makes the explanation valid. How many
+   Precondition: ServerVal = V                                       conjectured events might we need to insert? First of all, note
+                 Conflicts = C                                       that each U P event makes a DIRTY node CLEAN (and neither
+                 for all N, Fresh?N = FRESH                          U P nor D OWN can make a CLEAN node DIRTY). So, if there
+                            Clean?N = CLEAN                          are a total of N nodes, then at most N U P events can appear
+   Effect: none                                                      between consecutive observed events. Secondly, note that each
+                                                                     D OWN event makes a STALE node FRESH. So there can be at
+The intuition for this precondition is that this observed event is   most N D OWN events in a row. Since an U P event makes
+not considered valid unless the system model has also reached        N − 1 nodes STALE, each U P can be followed by up to
+a stable state. The precondition can be satisfied by adding          N − 1 D OWN events before another U P or an observed event
+conjectured upload and download actions to the explanation           must occur. Thus we need to insert at most N + N · (N − 1)
+until all nodes in the system state are FRESH and CLEAN.             conjectured events between each pair of consecutive ordered
+   The transition for failed stabilization events has an even        events; there are only finitely many possible explanations for
+stricter precondition: such an event is never allowed!               each observation, so it is decidable whether a test has passed.
+                                                                        In our implementation, we do not explore all possible
+    S TABILIZE → {(V1 , C1 ), . . . , (Vn , Cn )}                    explanations. We construct the set of possible states before and
+                                                                     after each observed event in an observation. Given the set of
+   Precondition: False                                               possible states before such an event, we select those satisfying
+   Effect: none                                                      the event’s precondition and apply the event’s action to them,
+                                                                     resulting in the set of possible states after the event. If the set
+This ensures that any observation including a failed stabiliza-
+                                                                     of possible states ever becomes empty, then the test fails.
+tion will be identified as a failing test case.
+                                                                        Now, given the set S of possible states after an observed
+   These are all the observed events. But we also need transi-       event, we construct the set of states before the following one
+tions for the conjectured events U P and D OWN. Downloading          by taking the image of S under the transitive closure of U P and
+a value from the server to a client node stores the server’s         D OWN . In any such state, (a) every node will have a LocalVal
+value as the local value for that node in the system state;          drawn from the set V of all LocalVals plus the ServerVal in
+it also changes the node from STALE to FRESH. However, it            the given state, (b) the ServerVal will also be an element of
+can only be performed if the client node is currently CLEAN.         V , (c) the Conflicts will be the union of the Conflicts in
+(Otherwise, it must be preceded by an U P event, which will          the given state, and a subset of V , and (d) each node will be
+reconcile the local value with the server’s.)                        FRESH or STALE, CLEAN or DIRTY . Since the size of V is at
+                                                                     most N + 1, it follows that there can be at most (N + 1)N +1 ·
+    D OWNN                                                           2N +1 · 4N states reachable from S.
+   Precondition: Fresh?N = STALE                                        Should this set become too large to deal with during testing,
+                 Clean?N = CLEAN                                     a pragmatic solution would be to abandon that test case and
+   Effect: LocalVal N ← ServerVal                                    generate another. We conjecture that most bugs can be found
+           Fresh?N ← FRESH                                           by a relatively deterministic test, so we would not expect this
+                                                                     solution to make many interesting bugs impossible to find.
+  The most interesting case is the U P transition. Here is a            However, in our experiments, N was at most 3, giving a
+simple first attempt (we will refine it below):                      bound of at most 262144 states reachable from each state—
+a large but not unmanageable number. In practice, we have             that no further changes would be required. But QuickCheck
+almost never seen more than 1,000 different possible states           soon finds this failing test case:
+during a test. Synchronizers are so slow that there is plenty of       Client 1               Client 2           Client 3
+time to compute 1,000 model states after each observed event!
+                                                                       W RITE1 ‘a’ → ⊥
+Refining the model: repeated values                                                           R EAD2 → ‘a’
+                                                                       W RITE1 ⊥ → ‘a’
+  Perhaps not surprisingly, with the first-draft model as
+                                                                                              R EAD2 → ⊥
+presented above, testing against Dropbox fails immediately.
+                                                                                                                 W RITE3 ‘b’ → ‘a’
+QuickCheck reports the following minimal counterexample:
+                                                                       R EAD1 → ‘b’
+            Client 1               Client 2
+                                                                      Why does this test fail? Because: at step 4, the server value
+            W RITE1 ‘a’ → ⊥
+                                                                      must be ⊥, since client 2 saw ⊥ after client 1 wrote it (and
+                                W RITE2 ‘a’ → ⊥
+                                                                      client 2 previously read the value ‘a’, so client 2 is not simply
+                     S TABILIZE → (‘a’, ∅)                            reading the initial ⊥); at step 5, because the value client 3
+The test fails because the two writes conflict—neither saw the        overwrites is not the server value, a conflict is created; the
+value written by the other. Our model says that both nodes            value ‘b’ should thus only appear in conflict files on other
+must upload their values before the S TABILIZE and that, on           nodes, never as the value in the file itself—yet it does just
+the second U P required to enable it, the value should be added       that in the last step. Thus, there can be no explanation for this
+to the set of conflicts. Yet the set of conflicts is observed to be   observation.
+empty at the end. The Next function as defined above admits              (This test is, of course, quite sensitive to timing. For
+no valid explanations of this observation.                            example, the second operation (R EAD2 → ‘a’) reads the
+   Evidently, Dropbox considers that there is no conflict if the      value written by the first W RITE1 ‘a’ → ⊥. This is only
+same value is written independently by two clients. This is a         possible if enough time passes after the first W RITE to allow
+sensible design decision, but it needs to be reflected in our         the synchronizer to act. The actual test case includes S LEEP
+specification. To make the implementation and specification           operations recording the need for these pauses, but they are
+agree, we need to refine the specification to add special cases       not shown in the observations we present. We will return to
+in the U P event when the local and global values are identical:      the question of timing in more detail in Section VI.)
+                                                                         We have discovered another inconsistency with our model,
+    U PN                                                              but not yet a bug: in fact, the behavior we are seeing reflects
+   Precondition: Clean?N = DIRTY                                      another sensible design decision—that when a delete and
+   Effect: Clean?N ← CLEAN                                            a write conflict, the write should take precedence (and the
+           if Fresh?N = FRESH then                                    deletion should be silently forgotten). We must amend our
+              if LocalVal N 6= ServerVal then                         model to reflect this too:
+                 Fresh?N 0 ← STALE for all N 0 6= N
+                                                                          U PN
+                 ServerVal ← LocalVal N
+           else                                                          Precondition: Clean?N = DIRTY
+              if LocalVal N 6= ServerVal then                            Effect: Clean?N ← CLEAN
+                 Conflicts ← Conflicts ∪ {LocalVal N }                           if Fresh?N = FRESH then
+                                                                                    if LocalVal N 6= ServerVal then
+With this modification, the test case passes. Here is a (newly                         Fresh?N 0 ← STALE for all N 0 6= N
+valid) explanation for the observation we made:                                        ServerVal ← LocalVal N
+            Client 1               Client 2                                      else
+            W RITE1 ‘a’ → ⊥                                                         if LocalVal N 6∈ {ServerVal , ⊥} then
+            U P1                                                                       Conflicts ← Conflicts ∪ {LocalVal N }
+                                W RITE2 ‘a’ → ⊥                       The change is in the second-to-last line, which now states that
+                                U P2                                  neither uploading the same value as the stable value, nor a
+                     S TABILIZE → (‘a’, ∅)                            deletion, ever generates a conflict. With this change, we believe
+Adding deletion to the model                                          our model reflects the intended behavior of the synchronizers
+                                                                      we have tested.
+   Since we already model reading from a missing file by the
+special value ⊥, deletion can be modelled simply as writing ⊥                                 V. S URPRISES
+back to the file. Of course, when executing tests we actually           What about unintended behaviors?
+implement this by performing a real file deletion, but the event
+that we observe is just W RITEN ⊥ → Vold , where Vold is              Dropbox Surprises
+the contents of the file just before deletion. Since the model          Up to this point, we were essentially debugging our model
+already encompasses W RITE events, we might have expected             using Dropbox as a reference implementation. However, con-
+tinued testing revealed further inconsistencies between our                     Here the file is created on one client, synchronized to the other,
+model and Dropbox.                                                              and overwritten there—but Dropbox does not copy the new
+   The first surprise was that Dropbox can (briefly) delete a                   value to the other client, and so the system never becomes
+newly created file:                                                             stable. Further investigation shows that Client 1 behaves as
+                                                                                though it is still FRESH, rather than STALE, so Client 2 never
+             Client 1                   Client 2
+                                                                                sees the value that Client 1 wrote, and if Client 2 writes
+             W RITE1 ‘a’ → ⊥
+                                                                                another value to the file then it is just copied onto Client 1—
+             W RITE1 ⊥ → ‘a’
+                                                                                no conflict is detected, and the value ‘a’ is lost forever.
+                                        W RITE2 ‘b’ → ‘a’
+                                                                                Again, the behavior is timing dependent, occuring when the
+             W RITE1 ‘c’ → ⊥
+                                                                                second W RITE happens very soon after the value ‘b’ arrives
+             R EAD1 → ⊥
+                                                                                on Client 1.
+In this case, W RITE2 ‘b’ and W RITE1 ‘c’ are in conflict;                         Dropbox offered the following response to these surprises:
+the final R EAD1 should see one of these values, with the                       Our engineers thoroughly investigated the file and data issues
+other eventually appearing in a conflict file—but at the time                   and were able to reproduce them programatically. Fortunately,
+we try to read the file, it is not there at all! In this case                   we don’t believe these issues have occurred outside of the lab
+stabilization would restore a correct file contents, but the test               due to the precise conditions necessary for any data loss to
+fails because our model does not allow the file to be missing,                  occur–namely, the local edit occurring within the same second
+even briefly. (Of course, observing this transient behavior                     as the remote change and the saved file having the identical
+requires executing the operations at just the right times; the                  file size as the original. While that fact gives us comfort,
+test case found by QuickCheck includes S LEEP operations that                   Dropbox takes any potential data issue, no matter how remote
+make this more likely.)                                                         in possibility, extremely seriously and we are developing fixes
+    The second surprise was that Dropbox can re-create deleted                  for the issue. We are grateful to the researchers for their
+files, even when only one client is modifying the file!4                        efforts in testing the Dropbox service using property-based
+                                                                                testing and raising awareness of property-based testing within
+                               Client 1
+                                                                                Dropbox.
+                           W RITE1 ‘b’ → ⊥
+                           W RITE1 ⊥ → ‘b’
+                            R EAD1 → ‘b’                                        ownCloud Surprises
+
+In this case Dropbox does not later ‘correct the mistake’ by                       While most of our effort has been spent testing Dropbox, we
+deleting the file again: it remains there permanently. (Again,                  have also used our model (unchanged) to test ownCloud and
+of course, Dropbox does not always restore files after they                     Google Drive. So far, Google Drive has behaved as expected,
+have been deleted: to provoke this behavior, the test case must                 but we elicited some surprising behavior from ownCloud.
+include a S LEEP operation of just the right length.)                              The first surprise was that ownCloud can delete newly
+   A similar test shows that deleted files can reappear even if                 created directories, instead of propagating them to other
+the creation and deletion take place on different nodes:                        nodes. More surprising yet, we actually discovered this when
+                                                                                our test setup failed! To mitigate the risk of one test case
+             Client 1                   Client 2                                interfering with the next, we run each test in a new directory.
+                                        W RITE2 ‘b’ → ⊥                         We create these directories in batches, to reduce the time
+              W RITE1 ⊥ → ‘b’                                                   spent waiting for them to propagate to all the nodes. At the
+              R EAD1 → ⊥                                                        start of a test run, we delete left-over test directories, then
+                     S TABILIZE → (‘b’, ∅)                                      recreate the ones we need. So, our preparation for a testing
+The R EAD1 → ⊥ verifies that the file was properly deleted;                     run looks like this: (1) On the host, delete all the left-over
+but, after waiting for the system to stabilize, it reappears.                   directories from previous tests. (2) On each virtual machine,
+  The most alarming behavior we observed shows that Drop-                       wait for the left-over directories to disappear. (3) On the
+box can lose data completely:                                                   host, create several hundred ‘fresh’ directories for the first
+                                                                                few hundred test cases to use. (4) On each virtual machine,
+             Client 1                    Client 2                               wait for all these new directories to appear. To our surprise,
+                                         W RITE2 ‘b’ → ⊥                        when using ownCloud as the synchronizer, step (4) often failed
+             W RITE1 ‘a’ → ‘b’                                                  to terminate. On checking progress, we found that not only
+             R EAD1 → ‘a’                                                       had the test directories not appeared on the virtual machines,
+                           S TABILIZE →                                         but they had been deleted from the host! We surmise that
+                 { (‘a’, ∅),           (‘b’, ∅) }                               ownCloud may arrange deletion followed by recreation of the
+                                                                                same directory in the wrong order, if they occur sufficiently
+   4 There were other machines logged in to the same Dropbox account, so
+                                                                                close together in time.
+Dropbox was synchronizing the file at the same time to these other machines.
+However, they were not involved in the test, and were not modifying the files      After we worked around this issue, QuickCheck found one
+in question; they were simply passive observers.                                further discrepancy between the specification and ownCloud’s
+actual behavior: ownCloud can lose changes. The following             just to allow a fixed time for synchronization to complete,
+observation illustrates what can happen:                              because the more file operations a test case performs, the
+                                                                      slower synchronization becomes. It appears that synchronizers
+          Client 1                 Client 2
+                                                                      ‘back off’ when files are changing rapidly, waiting for a
+          W RITE1 ‘a’ → ⊥
+                                                                      more opportune moment to do their job. This was the original
+                              W RITE2 ‘b’ → ‘a’
+                                                                      reason for including stabilization operations in our test cases,
+          W RITE1 ‘c’ → ‘a’
+                                                                      combining observations from all of the virtual machines to try
+                   S TABILIZE → (‘b’, ∅)
+                                                                      to detect when the synchronizer has nothing left to do.
+At the end, all clients have stabilized on the value ‘b’, while          The examples we presented above are minimized test cases,
+‘c’ has been completely forgotten even though it was written          found by QuickCheck’s shrinking search. Shrinking tries to re-
+independently from ‘b’ (both writes saw the previous value            duce the size of test cases by dropping calls from the sequence,
+‘a’) and, according to the specification, it should at least          but also reducing the duration of SLEEP operations—shrunk
+appear in the final conflict set. This time, we could confirm the     test cases should wait no longer than necessary to provoke
+reason by reading the ownCloud source code. The ownCloud              a failure. We also noticed that counterexamples leading to
+client uses a simple test for when a file has been changed and        wrong file contents were found in two forms: ending in a
+needs to be uploaded to the server: it checks whether either the      READ operation, or ending in a deletion (a W RITE of ⊥),
+file’s modification time or its length are different from their       which also observes the contents. We configured shrinking so
+last seen values. But modification times are recorded in the          that deletions shrink to R EAD operations (provided the test
+filesystem with a 1-second granularity. So if the file is written     still fails, of course), so that the latter form would shrink to
+twice in quick succession (i.e., during the same second) and          the former.
+the new contents is the same length as the old one, no change            Because of the non-determinism inherent in the system,
+will be detected. Thus, ‘b’ is recorded as the file’s stable value:   failing test cases may not fail every time they are run. This is
+no matter how long we wait, the value (‘c’) will never reach          problematic both when searching for a failing test case, and
+the server or Client 2. If the next write to the file occurs on       when shrinking one. Our solution in both cases is to run each
+Client 2, ‘c’ will be silently overwritten.                           test several times, and consider the test to have failed if any
+                                                                      of the runs fails—thus reducing the probability of a ‘false
+                VI. P RAGMATICS OF T ESTING                           negative’ result. While running random tests, we repeated
+   Many of the behaviors we encountered were timing depen-            each test three times, but while shrinking test cases, we
+dent (we have not included timings here, since their values will      repeated each one twenty times. (We work harder to avoid false
+vary with factors like connection speed). Our main technique          negatives during shrinking, because they lead QuickCheck to
+for provoking timing-dependent behaviors was to include               report non-minimal failing tests, which can waste a great deal
+S LEEP operations in our tests, which cause the whole testing         of human debugging time. In consequence, shrinking a failed
+framework to pause for a specific period (up to one second,           test to a minimal one can take 10–20 minutes to finish, at 10–
+randomly chosen during test generation).                              15 seconds per test. Twenty repetitions was usually enough to
+   We found that timing-dependent tests often failed with             minimize failing tests.)
+fairly low probability, which we could increase manually—
+once we’d identified a test case that sometimes failed—by                  VII. A N A LTERNATIVE S PECIFICATION ATTEMPT
+‘triggering’ some of the W RITE operations on changes in the             There is an appealing analogy between file synchronization
+file. That is, we busy-wait on some node until a specified            services and the memory subsystems of modern multiproces-
+value appears in the file and then immediately execute an             sors. Network hosts with copies of a replicated file correspond
+operation. Triggering an operation makes unexpected behavior          to individual processor cores, the local filesystems correspond
+more likely, since it may create a race condition between             to per-core caches, and the central synchronization server
+the test code running on node N , and the synchronization             corresponds to the main memory. This suggests that one might
+dæmon on that node. For example, it allowed us to observe             try to leverage ideas from the literature on specifications of
+the last two Dropbox surprises quite repeatably. It would be          memory-system behavior (see [9], [10] for surveys) to specify
+interesting to go a step further and automatically generate           the desired behavior of a synchronizer. In particular, perhaps a
+triggered operations as part of test cases; this would require        specification could be based on Lamport’s notion of happened
+a slightly richer generation-time state so that we can predict        before relations [11], which express the causal ordering of
+what value(s) might appear in the file.                               events in a distributed system. Indeed, an early version of our
+   We found the slowness of file synchronizers to be quite            QuickCheck specification was written in this style, rather than
+a problem; also the unpredictability of synchronization time.         the model-based style that we have described in this paper.
+It is not easy to tell when synchronization is complete—in               This early specification used the same notions of tests and
+particular, the icons that synchronizers display to show their        observed events as our current one. But instead of trying
+status are often wrong: the local dæmon itself is confused            to match the observed behavior against the transitions of
+about what state things are in! Yet we must know, if we are           a concrete model of the system (including the server), it
+to detect synchronization failures reliably. It does not work         directly specified which observations were legal by attempting
+to construct a partial ordering ≺ on the observed events such       directories—in particular, they spend considerable effort on
+that (1) if an event e happened before e0 on the same client,       the subtleties of conflicts in this setting. However, they address
+then e ≺ e0 , and (2) if e is the W RITE event that writes the      a different distribution scenario, in which the execution of
+value observed by e0 , then e ≺ e0 . If such a relation exists,     the synchronizer is a visible user-initiated action rather than
+then it can be taken as an explanation for the observation. For     a continuous background activity and in which there is no
+example, the validity of the observation                            centralized “global value.” They have not been used for testing.
+                                                                       We have implicitly assumed that the local filesystem is
+      Client 1                    Client 2
+                                                                    behaving correctly (so that discrepancies between our model
+      e1 : W RITE1 ‘a’ → ⊥
+                                                                    and actual observations are attributable to Dropbox). Ridge et
+      e2 : W RITE1 ‘b’ → ‘a’
+                                                                    al. [13] show how this can be tested, using a specification with
+                                  e3 : W RITE2 ‘c’ → ‘b’
+                                                                    significant similarities to ours.
+is justified by this relation e1 ≺ e2 ≺ e3 . On the other hand,        A distinct body of specification work deals with specifying
+if no ≺ relation exists that is consistent with the observations,   the behavior of operational transform services—middleware
+then a bug (or at least a discrepancy between the system and        layers that maintain consistency of replicated data structures
+the spec) has been detected. For example, the observation           (databases, documents, spreadsheets, etc.) under concurrent
+             Client 1               Client 2                        updates. Operational transform algorithms are widely used—
+             W RITE1 ‘a’ → ⊥                                        for example, they underlie behind the collaboration features
+             W RITE1 ‘b’ → ‘a’                                      in Apache Wave and Google Docs—and their theory is well
+                                     R EAD2 → ‘b’                   developed [14]–[16, etc.]. However, although it has been used
+                                     R EAD2 → ‘a’                   for debugging replication algorithms using symbolic model-
+                                                                    checking [17], the theory has not, to our knowledge, been
+cannot be partially ordered in a way that respects the two          applied to testing of actual distributed implementations. In-
+conditions above. To deal with conflicts, we specified that,        deed, since these specifications are based on notions of causal
+at stabilization points, all of the maximal values in the partial   ordering, our experiences reported in Section VII suggest that
+order (that is, the values written by every W RITE event e such     it might be difficult to do so, at least in a black-box style.
+that for no W RITE event e0 do we have e ≺ e0 ) must appear            Fraser and Wotowa [18] present a model-based testing
+either as the local value or in the conflict set on all nodes.      method for non-deterministic systems, using a model-checker
+   Unfortunately, although this style of specification at first     to generate test cases (sequences of transitions in the model)
+appeared quite natural and elegant, we found it difficult to        which fulfill selected structural coverage criteria. But when test
+extend to encompass all the behaviors we cared about. In            cases are run, the implementation may choose to make differ-
+particular, the fact that the same value can be written many        ent transitions, because of non-determinism. If the implemen-
+times during the same test run (e.g., the value ⊥ is written        tation diverges from the model at a deterministic point, then
+every time a file is deleted), renders the second condition         the test fails, but if divergence occurs at a non-deterministic
+above—“if e is the W RITE event that writes the value observed      choice point, then the test is considered inconclusive. Fraser
+by e0 ...”—impossible to test with certainty.                       and Wotowa show how to take inconclusive tests and generate
+   We tried dealing with this indeterminacy by constructing         new branches, again using the model checker, that fulfill
+≺ relations for all possible ways of matching W RITES with          the selected coverage goal, starting from the state that the
+later observations (and accepting a test case if we succeeded       implementation chose. The test is repeated, and if the imple-
+for any one of them), but the result was tricky to implement        mentation follows either the original or the newly generated
+and slow because the set of possibilities quickly became            path, then the coverage goal is reached. If the implementation
+large. Fortunately, this failed attempt gave us the idea of         diverges again, then the test is still inconclusive, and another
+working with limited knowledge about what the system is             branch can be added in the same way. Tests generated in
+doing by explicitly maintaining sets of possibilities, leading      this way are tree-structured, and hopefully eventually the
+to the current model-based specification.                           implementation will follow one of the paths in the tree, and
+                                                                    the coverage goal will be reached.
+                    VIII. R ELATED WORK                                Arcaini et al. [19] generate tests with a model checker in
+   As far as we know, this is the first work to address testing     a similar way, but instead of introducing branches, they reuse
+a distributed synchronization service. But the problems of          the model as a run-time monitor, to check that even if the
+specifying the behavior of synchronizers and of testing the         system follows a different path from the test case, then its
+behavior of nondterministic and distributed systems have both       input-output behavior still conforms to the model. They do
+received considerable attention.                                    assume that the inputs in the generated test can be supplied to
+   A series of formal specifications of the Unison file syn-        the SUT even though it is following an unexpected path, and
+chronizer [12] by Pierce and Balasubramaiam [1], Pierce             they assume that outputs from the SUT always provide enough
+and Vouillon [2] and Ramsey and Csirmaz [3] were the                information to uniquely identify the corresponding model state
+starting point for the present work. Those specifications go        (‘strong conformance’). They evaluate their approach using a
+further than ours in that they deal with multiple files and         Tic-Tac-Toe game, in which the model requires moves to be
+valid, but does not specify which moves the computer player          determinism in those papers is quite different to the one used
+should make. The Java implementation must be annotated in            here.
+order to link it to the model. No errors were discovered in the
+Tic-Tac-Toe implementation (but neither were any expected).                                  IX. F UTURE W ORK
+   In comparison, we use two state machines, a trivial one              We have considered just the case of a single file. Naturally,
+for generating tests, and a more interesting one as a run-time       there are interesting questions to ask about a synchronizer’s be-
+monitor. Our monitoring state machine is deterministic, but          havior in the presence of multiple files and directory structures,
+includes unobservable transitions—eliding those transitions          such as “what happens when a directory is deleted on one
+makes it non-deterministic. We allow multiple possible model         client, while a file is written into that directory on another?”
+states during monitoring (‘weak conformance’ in the sense of            Extending the testing framework to multiple files and di-
+[19]), and we treat the SUT as a black box—there is no need          rectories will require slightly richer model states at test case
+for instrumentation of the implementation to connect it to our       generation time, including the paths that have been created
+model. Our examples are more complex real applications, and          so far, so that operations can stay within this set with high
+we found a number of unexpected behaviors.                           enough probability to provoke bugs.
+   Ulrich and König propose architectures for testing dis-             One challenge that can be expected when we make this
+tributed systems [20], but assume that all internal actions of the   extension is that the set of possible system states given a
+SUT are observable by the tester, and that software probes are       sequence of observed events is likely to grow much more
+inserted into the SUT to allow the tester to control the timing      quickly (it will be exponential in the number of files), and
+of communications between nodes, and ensure that test runs           we will probably need to find clever representations for this
+are deterministic. Neither assumption holds in our setting.          set. Possibilities include representing the set as a cartesian
+   Boy et al. report on an approach to testing servers by            product of smaller sets—we can overapproximate the set of
+running random sequences of API calls from a number of               possible states without introducing false positives, so ideas
+clients, and checking that specified invariants hold over the        from abstract interpretation [32] should be applicable here.
+resulting traces [21]. The invariants are specified as patterns         Another rich source of incorrect behaviors in distributed
+that are matched against the traces, and assertions that must        systems is network partitions. To provoke such behaviors, it
+hold if a pattern matches. Boy et al. found a subtle timing bug      might be useful to extend our test cases with operations to
+in a lock server using this method. The approach does not            disconnect and reconnect hosts from the network.
+address ‘conjectured events’, however, and does not include
+shrinking—the lock-server bug was minimized by hand.                                          X. C ONCLUSIONS
+   A variety of work on testing nondeterministic systems can            We have described an executable formal specification of
+be phrased in terms of the theory of input-output conformance        the core behavior of file synchronization services. Since it’s
+(ioco) testing [22], [23]. Indeed, there are some suggestive         written in a black-box style, avoiding synchronizer-specific
+similarities between aspects of this theory and the structures       APIs and communicating only via the filesystem, we were able
+we used in specification and testing of synchronizers—e.g.,          to apply it to three popular synchronizers—two commercial,
+the inclusion in its labeled transition systems of quiescence        one open source—and found surprising behaviors in two of
+transitions, which are reminescent of our stabilization events.      them. This shows the effectiveness of the method.
+It would be interesting to try to reframe our development in
+                                                                        Given that three different synchronizers appear to share the
+terms of ioco concepts.
+                                                                     same core specification, we expect that our model should be
+   QuickCheck was originally developed in and for Haskell
+                                                                     applicable (perhaps with small changes to the testing frame-
+[5], and it has become the most widely used testing tool
+                                                                     work) to many others—e.g., Microsoft OneDrive, Box.net,
+in that community. The version we used was developed
+                                                                     SpiderOak, Sugarsync, Seafile, Pulse, Wuala, Teamdrive,
+by Quviq and supports Quviq’s core business: specification-
+                                                                     Cloudme, Cx, etc. Given the surprising behaviors already
+based testing tools and services. Quviq QuickCheck [24]
+                                                                     found, this should be a valuable exercise.
+extends the original version with libraries tailored for testing
+industrial software, such as the state machine library used                                 ACKNOWLEDGMENTS
+here. It has been used to test many large systems, including
+telecoms products [4], a messaging gateway [25], refactoring            We thank John Lai and other Dropbox engineers for their
+tools [26], [27], and quadcopters [28]. Probably the largest         feedback. This work is partially funded by the EU FP7
+application so far was to test AUTOSAR basic software (C             project PROWESS (#317820), the Swedish Strategic Research
+code which runs in cars), in which a million lines of C              Foundation (RAWFP), and the National Science Foundation
+was tested against 3,000 pages of the AUTOSAR standard,              (CCF-1421243).
+using 20,000 lines of QuickCheck code [29]. Most of these                                         R EFERENCES
+systems are deterministic, but QuickCheck has also been used
+to test for race conditions in concurrent programs [30], finding      [1] S. Balasubramaniam and B. C. Pierce, “What is a file synchronizer?” in
+                                                                          Fourth Annual ACM/IEEE International Conference on Mobile Comput-
+two long-standing race conditions in the database distributed             ing and Networking (MobiCom ’98), Oct. 1998, full version available
+with Erlang [31]. However, the approach to handling non-                  as Indiana University CSCI technical report #507, April 1998.
+ [2] B. C. Pierce and J. Vouillon, “What’s in Unison? A formal specification        [23] ——, “Model based testing with labelled transition systems,” in Formal
+     and reference implementation of a file synchronizer,” Dept. of Computer             methods and testing. Springer, 2008, pp. 1–38. [Online]. Available:
+     and Information Science, University of Pennsylvania, Tech. Rep. MS-                 http://liacs.leidenuniv.nl/∼bonsanguemm/Toos/P9 TestingTransSyst.pdf
+     CIS-03-36, 2004.                                                               [24] J. Hughes, “Software testing with quickcheck,” in Proceedings of the
+ [3] N. Ramsey and E. Csirmaz, “An algebraic approach to file synchro-                   Third Summer School Conference on Central European Functional
+     nization,” in Proceedings of the 8th European Software Engineering                  Programming School, ser. CEFP’09. Berlin, Heidelberg: Springer-
+     Conference. ACM Press, 2001, pp. 175–185.                                           Verlag, 2010, pp. 183–223. [Online]. Available: http://dl.acm.org/
+ [4] T. Arts, J. Hughes, J. Johansson, and U. Wiger, “Testing telecoms                   citation.cfm?id=1939128.1939134
+     software with quviq quickcheck,” in Proceedings of the 2006                    [25] J. Boberg, “Early fault detection with model-based testing,” in
+     ACM SIGPLAN Workshop on Erlang, ser. ERLANG ’06. New                                Proceedings of the 7th ACM SIGPLAN Workshop on ERLANG, ser.
+     York, NY, USA: ACM, 2006, pp. 2–10. [Online]. Available:                            ERLANG ’08. New York, NY, USA: ACM, 2008, pp. 9–20. [Online].
+     http://doi.acm.org/10.1145/1159789.1159792                                          Available: http://doi.acm.org/10.1145/1411273.1411276
+ [5] K. Claessen and J. Hughes, “Quickcheck: A lightweight tool for                 [26] D. Drienyovszky, D. Horpácsi, and S. Thompson, “Quickchecking
+     random testing of haskell programs,” in Proceedings of the Fifth ACM                refactoring tools,” in Proceedings of the 9th ACM SIGPLAN Workshop
+     SIGPLAN International Conference on Functional Programming, ser.                    on Erlang, ser. Erlang ’10. New York, NY, USA: ACM, 2010, pp. 75–
+     ICFP ’00. New York, NY, USA: ACM, 2000, pp. 268–279. [Online].                      80. [Online]. Available: http://doi.acm.org/10.1145/1863509.1863521
+     Available: http://doi.acm.org/10.1145/351240.351266                            [27] H. Li and S. Thompson, “Implementation and application of functional
+                                                                                         languages,” O. Chitil, Z. Horváth, and V. Zsók, Eds. Berlin,
+ [6] A. Zeller and R. Hildebrandt, “Simplifying and isolating failure-
+                                                                                         Heidelberg: Springer-Verlag, 2008, ch. Testing Erlang Refactorings
+     inducing input,” IEEE Trans. Softw. Eng., vol. 28, no. 2, pp. 183–200,
+                                                                                         with QuickCheck, pp. 19–36. [Online]. Available: http://dx.doi.org/10.
+     Feb. 2002. [Online]. Available: http://dx.doi.org/10.1109/32.988498
+                                                                                         1007/978-3-540-85373-2 2
+ [7] J. Hughes, “Quickcheck testing for fun and profit,” in Proceedings of          [28] B. Vedder, J. Vinter, and M. Jonsson, “Using simulation, fault injection
+     the 9th International Conference on Practical Aspects of Declarative                and property-based testing to evaluate collision avoidance of a quad-
+     Languages, ser. PADL’07, 2007, pp. 1–32.                                            copter system,” in Dependable Systems and Networks Workshops (DSN-
+ [8] U. Norell, H. Svensson, and T. Arts, “Testing blocking operations                   W), 2015 IEEE International Conference on, June 2015, pp. 104–111.
+     with quickcheck’s component library,” in Proceedings of the Twelfth            [29] T. Arts, J. Hughes, U. Norell, and H. Svensson, “Testing autosar soft-
+     ACM SIGPLAN Workshop on Erlang, ser. Erlang ’13. New                                ware with quickcheck,” in Software Testing, Verification and Validation
+     York, NY, USA: ACM, 2013, pp. 87–92. [Online]. Available:                           Workshops (ICSTW), 2015 IEEE Eighth International Conference on,
+     http://doi.acm.org/10.1145/2505305.2505310                                          April 2015, pp. 1–4.
+ [9] S. V. Adve and K. Gharachorloo, “Shared memory consistency models:             [30] K. Claessen, M. Palka, N. Smallbone, J. Hughes, H. Svensson,
+     A tutorial,” computer, vol. 29, no. 12, pp. 66–76, 1996.                            T. Arts, and U. Wiger, “Finding race conditions in erlang with
+[10] L. Higham, J. Kawash, and N. Verwaal, “Defining and comparing                       quickcheck and pulse,” in Proceedings of the 14th ACM SIGPLAN
+     memory consistency models,” in In Proc. of the 10th Int’l Conf. on                  International Conference on Functional Programming, ser. ICFP ’09.
+     Parallel and Distributed Computing Systems, 1997, pp. 349–356.                      New York, NY, USA: ACM, 2009, pp. 149–160. [Online]. Available:
+[11] L. Lamport, “Time, clocks, and the ordering of events in a distributed              http://doi.acm.org/10.1145/1596550.1596574
+     system,” Communications of the ACM, vol. 21, no. 7, pp. 558–565, 1978.         [31] J. M. Hughes and H. Bolinder, “Testing a database for race
+[12] B. C. Pierce, T. Jim, and J. Vouillon, “U NISON: A portable, cross-                 conditions with quickcheck: None,” in Proceedings of the 10th
+     platform file synchronizer,” 1999–present, http://www.cis.upenn.edu/                ACM SIGPLAN Workshop on Erlang, ser. Erlang ’11. New
+     ∼bcpierce/unison.                                                                   York, NY, USA: ACM, 2011, pp. 72–77. [Online]. Available:
+[13] T. Ridge, D. Sheets, T. Tuerk, A. Giugliano, A. Madhavapeddy, and                   http://doi.acm.org/10.1145/2034654.2034667
+     P. Sewell, “Sibylfs: formal specification and oracle-based testing for         [32] P. Cousot and R. Cousot, “Abstract interpretation: a unified lattice model
+     posix and real-world file systems,” in Proceedings of the 25th Symposium            for static analysis of programs by construction or approximation of
+     on Operating Systems Principles. ACM, 2015, pp. 38–53.                              fixpoints,” in Proceedings of the 4th ACM SIGACT-SIGPLAN symposium
+[14] C. A. Ellis and S. J. Gibbs, “Concurrency control in groupware systems,”            on Principles of programming languages. ACM, 1977, pp. 238–252.
+     in Acm Sigmod Record, vol. 18, no. 2. ACM, 1989, pp. 399–407.
+[15] D. B. Terry, M. M. Theimer, K. Petersen, A. J. Demers, M. J. Spreitzer,
+     and C. H. Hauser, Managing update conflicts in Bayou, a weakly
+     connected replicated storage system. ACM, 1995, vol. 29, no. 5.
+[16] Y. Saito and M. Shapiro, “Optimistic replication,” ACM Computing
+     Surveys (CSUR), vol. 37, no. 1, pp. 42–81, 2005.
+[17] H. Boucheneb, A. Imine, and M. Najem, “Symbolic model-checking
+     of optimistic replication algorithms,” in Integrated Formal Methods.
+     Springer, 2010, pp. 89–104. [Online]. Available: https://hal.inria.fr/inria-
+     00524535/document
+[18] G. Fraser and F. Wotawa, “Test-case generation and coverage analysis
+     for nondeterministic systems using model-checkers,” in Software En-
+     gineering Advances, 2007. ICSEA 2007. International Conference on.
+     IEEE, 2007, pp. 45–45.
+[19] P. Arcaini, A. Gargantini, and E. Riccobene, “Combining model-based
+     testing and runtime monitoring for program testing in the presence
+     of nondeterminism,” in Software Testing, Verification and Validation
+     Workshops (ICSTW), 2013 IEEE Sixth International Conference on.
+     IEEE, 2013, pp. 178–187. [Online]. Available: http://citeseerx.ist.psu.
+     edu/viewdoc/download?doi=10.1.1.309.7963&rep=rep1&type=pdf
+[20] A. Ulrich and H. König, “Architectures for testing distributed systems,”
+     in Testing of Communicating Systems. Springer, 1999, pp. 93–108.
+[21] N. Boy, J. Casper, C. Pacheco, and A. Williams, “Automated testing
+     of distributed systems,” May 2004, final project report for MIT 6.824:
+     Distributed Computer Systems.
+[22] J. Tretmans, “Test generation with inputs, outputs and repetitive
+     quiescence,” Software—Concepts and Tools, no. TR-CTIT-96-26,
+     1996. [Online]. Available: http://doc.utwente.nl/65463/1/Tre96-CTIT96-
+     26.pdf
+

packages/opencode/specs/simulation-research/text/2022-pbt-for-metamorphic-testing.txt

@@ -0,0 +1,508 @@
+                                                                  Application of property-based testing tools
+                                                                           for metamorphic testing
+
+                                                                      Nasser Alzahrani, Maria Spichkova, James Harland
+                                                                 School of Computing Technologies, RMIT University, Melbourne, Australia
+                                                              s3297335@student.rmit.edu.au, {maria.spichkova,james.harland}@rmit.edu.au
+
+
+
+
+                                         Keywords:     Software Testing, Metamorphic Testing, Property-Based Testing, Formal Specification
+
+                                         Abstract:     Metamorphic testing (MT) is a general approach for the testing of a specific kind of software systems –
+arXiv:2211.12003v1 [cs.SE] 22 Nov 2022
+
+
+
+
+                                                       so-called “non-testable”, where the “classical” testing approaches are difficult to apply. MT is an effective
+                                                       approach for addressing the test oracle problem and test case generation problem. The test oracle problem is
+                                                       when it is difficult to determine the correct expected output of a particular test case or to determine whether
+                                                       the actual outputs agree with the expected outcomes. The core concept in MT is metamorphic relations
+                                                       (MRs) which provide formal specification of the system under test. One of the challenges in MT is effective
+                                                       test generation. Property-based testing (PBT) is a testing methodology in which test cases are generated
+                                                       according to desired properties of the software. In some sense, MT can be seen as a very specific kind of PBT.
+                                                       In this paper, we show how to use PBT tools to automate test generation and verification of MT. In addition to
+                                                       automation benefit, the proposed method shows how to combine general PBT with MT under the same testing
+                                                       framework.
+
+                                                       Preprint. Accepted to the 17th International Conference on Evaluation of Novel Approaches to Soft-
+                                                       ware Engineering (ENASE 2022). Final version published by SCITEPRESS, http://www.scitepress.org
+
+
+
+                                         1    INTRODUCTION                                               on actual code which help in finding subtle faults on
+                                                                                                         live running systems [15]. One of the main attributes
+                                         Formal specification is an essential tool for managing          of PBT is that it can automatically generate tests to
+                                         the complexity of specifying and verifying the design           cover edge cases that are not so obvious to identify
+                                         and the development of critical software systems. The           manually. Two main elements of PBT approach make
+                                         formal approach removes ambiguity, improves preci-              this possible: (1) a random test generator, responsible
+                                         sion, and used to verify that the requirements are ful-         for generating random values in a controlled way, and
+                                         filled. Appel et al. summarised a number of desired             (2) a so-called shrinker, minimizing the number of the
+                                         qualities that the specification should have in order           generated tests cases to allow for easier debugging.
+                                         to be effective, see [3]. Firstly, the specification has            Metamorphic Testing (MT) is a special PBT tech-
+                                         to be formal, where the specification should be math-           nique elaborated for the cases where it’s complicated
+                                         ematically precise. It should be rich, i.e. precisely           to specify “classical” test cases having input and out-
+                                         expressing the intended behaviour of the system (we             put data flows - in some cases, it’s difficult to iden-
+                                         could reformulate this quality as completeness). The            tify what could be the correct output for each partic-
+                                         specification has to be two-sided where the specifica-          ular input. For “classical” testing we need to have a
+                                         tion is exercised by both implementations and clients.          so-called an oracle that can determine whether or not
+                                         Finally, the specification has to be live where it is au-       the output is correct wrt. the provided input and this
+                                         tomatically checked against actual code rather than             decision should be taken in a reasonable amount of
+                                         some abstract model.                                            time, see [30]. In the case an oracle cannot be cre-
+                                              Formal languages like TLA+ [18], or Alloy [16]             ated, the system are typically called non-testable (or
+                                         are generally concerned with specifying systems                 untestable), but MT can provide an effective solution
+                                         against some models rather than the actual code under           to the oracle problem using Metamorphic Relations
+                                         development. On the other hand, property-based test-            (MR), see e.g., [24]. MT was initially introduced in
+                                         ing (PBT) facilitates the use of formal specifications          [5] in the domain of numerical analysis. Since then, it
+has developed and been applied in many application           phic testing.
+areas, such as compilers, medical systems, embedded              In validity checks, one writes functions to check
+applications, search engines, service computing, sim-        some invariants of the system under test or the
+ulation software, image processing systems, machine          datatypes used in the system. This process also in-
+learning software and optimizing software [6].               cludes writing properties to check test-case generator
+     PBT tools for testing functional programs were          and test-case shrinker both produce valid results. The
+first introduced in the Haskell programming language         last step is to write property for the functions under
+by [11], where QuickCheck, a library for random test-        test which performs a single random call and checks
+ing of program properties, was implemented. Since            that the return value is valid. For instance, when test-
+then, many libraries have been developed following           ing the functionality of inserting a value into a tree
+this approach for different programming languages.           datatype, we demand that all the keys in a left subtree
+The main components of QuickCheck are the gener-             must be less than the key in the node, and all the keys
+ator of random values, the shrinker, and the checker         in the right subtree must be greater.
+which runs these random values with pre-selected                 A postcondition is a property that should be true
+functions. In our work, we extend the generator and          after a call. One can come up with such property by
+the shrinker in order to automate MT test generation         asking what would be the expected state of the system
+and verification.                                            after calling some function. A postcondition usually
+     The idea of generating random tests is not new,         tests one function after calling this function with some
+see for example [8]. However, using PBT tools such           random argument and then checking an expected re-
+QuickCheck has many advantages. First of all, PBT            lationship between the result and its argument. For
+has many tools for automating and controlling the            Instance, after calling insert (which inserts a value in
+generation of random test cases. Secondly, these tools       some tree datatype), we should be able to find the key
+allow controlled strategies for generating random data       inserted without changing previously inserted keys.
+of complicated data types, i.e., it is possible to config-       A model-based property is used to test some func-
+ure how the random values distribute over the input          tion by making a single call and comparing its result
+domain. Although [7] argues that one of MT’s main            with the result of some abstract operation. The model
+advantages is the ease of test case automation, the MT       refers to the abstraction functions which map the real
+automation is a complex task, when using PBT tools           arguments and results to abstract values.
+such as QuickCheck as a tool to test MR relations.               Inductive properties are the properties that one can
+     Contributions: The main contribution of this pa-        use induction to assert that the only function that can
+per is a systematic approach in which we utilize the         pass the tests is the correct one. This is usually done
+random generation of test cases and automatic test-          by relating a call of the function under test to calls
+ing capabilities of PBT tools to automate some steps         with smaller arguments. The set of inductive prop-
+of metamorphic testing: More precisely, we auto-             erties covering all possible cases allows the testing
+mate test case generation and test case verification by      of the base case and induction steps of an inductive
+extending QuickCheck’s shrinker and generator with           proof-of-correctness.
+our customized shrinker and generator. These variants            The canonical example in the literature to explain
+are simpler and more amendable to customization in           property based testing is the reversing a list. One
+the context of MT.                                           property that should hold for all lists is that reversing
+                                                             a list x twice returns the original list:
+                                                                              reverse (reverse x) = x
+2    BACKGROUND:                                             In the case of functional oracle-based testing, we
+     PROPERTY-BASED TESTING                                  would need to identify the equivalence partitions (the
+                                                             sets of inputs that have to be handled equivalently as
+Property-based testing (PBT) is an approach to test-         they have to provide the same type of output but with
+ing software by defining general specifications and          possibly different values), and then specify at least
+properties that must hold for all the executions of ran-     one test case for each partition, or use boundary val-
+domly generated test cases. The inputs to these test         ues in partitions that are ranges. Thus, for reverse
+cases are random. If these properties do not hold,           function this could be an empty list [] and some non-
+a minimized failing tests are reported. In PBT, test         empty list, e.g, [1, 2, 3], i.e., our test cases would be
+cases are generated randomly according to universally                   reverse (reverse []) = []
+quantified properties. Examples of quantified proper-                   reverse (reverse [1, 2, 3]) = [1, 2, 3]
+ties include; validity checks, postconditions, model-        In contrast to this approach, in PBT the checks take
+based properties, inductive properties, and metamor-         place on the return value of the function under test,
+instead of checking values “hard-coded” in the test         outputs. When implementing MT, we first generate
+cases such as [] and [1, 2, 3]. That is, the input to       source test cases. Then use the MR to generate new
+the function under test would be automatically gener-       input. This new input is then used to compare the
+ated and the library chooses random values for testing      output from the first set of the tests with the last ones.
+rather than the tester specifying particular values. For        In our proposed method, we show how to auto-
+example, for the reverse function, one could call the       mate the test case generation and verification of MT
+reverse function twice and expect the original list to      using QuickCheck. Our method can be generalized to
+be returned. The library generates many random test         work with other PBT tools other than QuickCheck as
+cases and reports a failure if it finds a counterexample.   our method extends features of QuickCheck that are
+    In the case of QuickCheck, the property to be           common in other PBT tools.
+passed to the library is reverse (reverse x) = x, the           Figure 1 illustrates MT in the context of testing
+library will generate the random values for x, and will     a compiler. There are two paths that are expected to
+report a failure if it finds a counterexample.              lead to the same output:
+    It is worth noting that there are some anti-patterns    (1) We start with Program 1, it is compiled to obtain
+that could emerge while writing property-based tests.           the corresponding executable code Executable 1.
+Because it is sometimes difficult to think about a              Then, we run Executable 1 with some input data
+property, practitioners usually fall into the trap of du-       x to get an output.
+plicating the implementation of the code in tests. The
+literature on PBT has many other examples of such           (2) We modify Program 1 to a semantically equiv-
+anti-patterns and how to avoid them. One solution to            alent but syntactically different Program 2 (e.g.,
+avoid this problem is MT.                                       by unrolling a loop or removing a comment etc.),
+    MT is a successful method in solving the oracle             then apply the same compilation method to get the
+problem in software testing. The core idea is this: in          corresponding executable code Executable 2. Af-
+the cases when it’s hard to specify in advance what             ter that, we run Executable 2 with the same input
+exactly should be the output of a function, we may be           data x to get an output.
+able to observe the change to the output when chang-        When both outputs are obtained, we compare them: if
+ing the input. That is, valuable information about the      they are not exactly the same, the compiler is faulty.
+function would be whether and how its input changes             One of the benefits of using PBT for MT is the
+when we change its input. For instance, even if the         rich sets of tools available. For instance, PBT tools
+expected result of a function such as inserting a key       allow the creation of strategies for generating random
+into a tree is difficult to predict, we may still be able   data for complicated data types with minimal setup.
+to express an expected relationship between this re-        In MT, the operations and data types are usually more
+sult and the result of some other call. In this case, if    complicated than simple data types such as integers
+we insert an additional key into t before calling insert    or lists. Existing approaches for creating these data
+k v, then we expect the additional key to appear in the     types are ad-hoc [9]. In these other approaches, one
+result also.                                                has to do almost the same setup work for every kind
+                                                            of data type in order to generate the random values.
+                                                            PBT tools, on the other hand, are more general and
+                                                            cover more data types without the need to duplicate
+                                                            the code for every kind of data type or model. In ad-
+                                                            dition, PBT requires less code than these other ap-
+                                                            proaches with more control over the distribution of
+                                                            the test cases space. These approaches evenly dis-
+                                                            tribute the test cases over the input space.
+                                                                Creating a random BST using any PBT library re-
+                                                            quires less setup work. One only required to define
+                                                            the data type and pass it to the library. However, since
+  Figure 1: Testing compilers using Metamorphic testing     we are using these PBT tools for MT, some more
+                                                            setup and customization are required.
+
+Metamorphic Relations (MRs) are a central element
+of MT. The MRs are properties of the function under
+test. An important (and usually missed) attribute of
+MR is that they relate multiple inputs to their expected
+3    RELATED WORK                                         4    PROPOSED APPROACH
+The effectiveness of MT in alleviating the oracle         In this section, we present a systemic method to
+problem has allowed it to appear in many different        use PBT tools to test MRs. We specifically choose
+application domains. However, many of these appli-        QuickCheck to illustrate the proposed approach. Our
+cations do not provide a systematic way to automate       method is general and can be implemented using
+some parts MT.                                            other PBT libraries as well. Our proposed method
+    The automation of MT was first introduced in          consists of three aspects. First, we develop a new gen-
+[13] where they proposed a framework that utilizes        erator for generating test cases for MT. Second, we
+Constraint Logic Programming techniques to find test      develop a new test case shrinker. Finally, we use the
+data that violate a given metamorphic relation. How-      newly designed generator and the shrinker instead of
+ever, they require the usage of special metamorphic-      QuickCheck’s default generator and shrinker. In the
+relations, such as permutation-based relations, to        rest of the section, we present the core features and
+speed up the search among the possible test data.         steps of our approach, and then discuss the advantages
+    There are few other efforts to automate MT steps.     of this approach.
+For instance, Zhu created a tool for automating meta-         QuickCheck is a library for random testing of pro-
+morphic testing for Java unit tests, see [32]. This       gram properties. The programmer provides a spec-
+method is specific to Java unit tests. Our method is      ification of the program, in the form of properties
+more general and can be applied in any programming        that functions should satisfy. The library then gener-
+language which has library support for PBT.               ates a large number of random test cases and checks
+    An automatic MT framework for compilers is pro-       that the property holds. Specifications are expressed
+posed in [29]. Their approach in generating the test      in Haskell. The Haskell programming language also
+cases is similar to the approach presented in this pa-    provides functions to define properties, observe the
+per. However, their approach is tailored to the domain    distribution of test data, and define test data genera-
+of testing compilers, where we propose a generally        tors, which is an important advantage for system spec-
+applicable solution.                                      ification.
+    In [20], the authors propose a method that allows         When using PBT tools such as QuickCheck there
+the composition of new metamorphic relations based        is some expected setup that needs to be done before
+on previously defined ones, their case study showed       defining the properties. One such setup is shrinking.
+that new metamorphic relations can be constructed         The main objective of shrinking is to produce a mini-
+by compositing some existing metamorphic relations.       mum failing test cases which facilitate the debugging
+They assert that the new derived metamorphic relation     of the program. Another required setup is the random
+delivers better metamorphic testing than the original     values generator which can be configured depending
+metamorphic relation as well as reduces the number        on the scenario. More importantly, the generator and
+of test cases.                                            shrinker need to be designed to work together when
+    Work related to verifying authenticated data struc-   testing MRs. Otherwise, if we use the default test
+tures (ADS) is presented in [23]. The approach of         case generator and shrinker, the checkers might miss
+Miller provides a semantics for a programming lan-        some test cases or generate invalid tests. The pro-
+guage LambdaAuth, which supports ADS. This ap-            posed method ensures that does not happen.
+proach provides many benefits, however, it might be           The steps needed to systematically test MR rela-
+hard to convince practitioners to use it which is less    tions using PBT tools are as follows:
+likely to be widely spread among engineers and is dif-    (1) Specify an MR property
+ficult to have an impact. In [4], the authors used Is-
+abelle proof assistant to formally verify LambdaAuth.     (2) Customize the test case generator
+They also assert that they found several mistakes in      (3) Customize the test case shrinker
+the semantics of lambdaAuth. In our work, we use          (4) Run the checker
+a mainstream programming language (Haskell) to de-
+sign such ADS and verify our implementation of these      Our main contributions are in Steps 2 and 3. Let us
+ADS using PBT and MT.                                     now discuss these steps and our solution in more de-
+                                                          tail.
+                                                               Step 1: Specify an MR property. Specifying
+                                                          suitable MRs is key in MT. Although identifying MRs
+                                                          is not a difficult task, this is typically a manual pro-
+                                                          cedure, see [21], and we don’t intend to automate
+                                                          it within our approach. However, there have been
+some approaches intending to automate this step [10],           is it uses partially defined test values. If a prop-
+and in our future work, we consider combining our               erty returns a Boolean result for a partially defined
+method with these approaches.                                   value, the shrinker does not enumerate more ver-
+     Step 2: Customize the test case generator. The             sions of this value. The benefit is that the checker
+first thing that all PBT libraries do is to generate ran-       will stop as soon it encounters the first failing test
+dom inputs for the functions under test. In the PBT             which improves the speed of the checker.
+literature, this is known as generation. For every type,        One of the differences between QuickCheck
+there is an associated random test generator.               shrinker and our shrinker is that our method of shrink-
+     For example, to generate a list of values, one has     ing is integrated into generation. It is worth noting
+to use the generator together with two parameters.          that almost all PBT tools in many different program-
+The first parameter is the number of elements in the        ming languages use a similar shrinking methodology
+list. The second parameter is the size which depends        as the one used in QuickCheck. The main problem
+on the type of values being generated and the con-          with this approach is that shrinking is defined based
+text. For example, the size can be the maximum value        on datatypes. This constraints the ways in which val-
+of Integer type, the maximum length of a list, or the       ues are shrinked. That is, there is only one way to de-
+depth of a binary search tree.                              fine shrinking for the same data type without taking
+     For MT, the generator has to be customized to pro-     into consideration the way it was generated. On the
+duce valid test cases. In Section 5 we will present an      other hand, our shrinker is composed with the gener-
+example of BST, where the values should not be gen-         ator and the generator controls how the values it pro-
+erated uniformly.                                           duces shrinks.
+     Step 3: Customize the test case shrinker. Al-              Our approach to shrinking has many benefits. For
+most all PBT libraries and frameworks have a mech-          example, shrinking happens even if there is no defined
+anism to reduce the set of generated test cases that        shrinker on the datatype. This allows the shrinker
+fails a property to a minimum number of failing test        to share the same variants as the generator and, at
+cases that is necessary for the debugging process, as       the same time, reduce the effort needed to write a
+an unnecessary large number of the failed tests cases       separate shrinker for each datatype involved in the
+(where many cases might refer to the same error) will       test. Another benefit of our shrinker is that failure
+make debugging process more complicated and time-           reported is more revealing than the shrinker defined
+consuming. This mechanism is known in the litera-           as datatype. For instance, in QuickCheck, errors are
+ture as shrinking.                                          sometimes shrinked to different errors, which is un-
+     Step 4: Run the checker. In this step, we pass         desirable since the error we expect is being reduced
+the MR property, which was specified in Step 1 to the       to another error we do not care about. To mitigate
+checker. If the function under test does not satisfy the    this problem, one has to duplicate the constraint logic
+MR, the PBT library will report the failing test case       both in the generator and in the shrinker. In our im-
+that violated the MR property.                              plemented shrinker, the main idea is that we shrink the
+                                                            outputs by shrinking the inputs. This help in finding
+As mentioned in Step 3, we cannot use the de-               possible more shrinks based on that representation.
+fault QuickCheck’s shrinker for testing MR proper-              Our designed shrinker covers the range between
+ties. Thus, we modify QuickCheck generator with our         the smallest value of some type and increases the
+designed generator. The default QuickCheck’s gener-         value until the test fails. It repeats this process until
+ator is based on [12] which is found to require some        the test passes. In this case, it reports the largest value
+efforts to use in MT. On this basis, we design a mod-       from the previous step as the smallest test case that
+ified generator and instruct QuickCheck to use it in-       fails the property, i.e., the boundary values. For exam-
+stead of its default one. The advantage of the pro-         ple, suppose that we are testing whether the value of
+posed approach is that the testing of MT can together       variable x of type Integer is less than 77 (x < 77). Sup-
+with other properties under the same testing frame-         pose that the first random value that is generated (by
+work. Thus, the same shrinker and verifier can be           the generator) is 90 which will cause the test to fail.
+used for both MT and general properties to test.            Then, the shrinker will generate new random values
+    Our version of shrinker has the following features:     and in random steps ranging from zero to 89. Now,
+ • The values are enumerated by depth instead of            maybe the new failing value is 89. The shrinker will
+   size and for this reason, the number of values           repeat the same process again for the values between
+   tends to grow quickly as our shrinker explores fur-      zero and 88. The shrinking repeats until the random
+   ther test cases.                                         value is 78 after which the smallest failing test value
+ • The modified shrinker exploits laziness [14]. That       is 77. After which the shrinker stops.
+    The way we ensure the validity of the generated        ues into a BST. Starting with the Tree at the top, we
+(then shrinked) test cases is by adding a precondition.    insert some key k1 and some value v1 to get some
+The main objective of a precondition is to inform the      modified tree. Then, another key k2 and value v2 is
+generator not to generate invalid test cases using the     inserted into the modified tree to get the out put tree
+valid function that we have to implement. The valid        (whatever it is). We repeat the same operation to the
+function checks the property before passing it to the      original tree but we change the inputs to insert. That
+generator. The generator will still generate random        is, we insert k2 and v2 followed by inserting k1 and v1
+test cases but they will not be executed. The valid        to get the out put tree. The metamorphic relation as-
+function depends on the context. For instance, in the      serts that the two out put trees should be the same oth-
+context of Binary Search Trees, the valid function         erwise insert is faulty. The notion of quality between
+checks that the keys in the left subtree are less than     two trees depends on the operations under test. For
+the key at the root node and all the keys on the right     insert, we can just assert that if the keys and values
+subtree is greater than the key at the root node.          in both trees are the same the trees are semantically
+                                                           equivalent.
+
+5    EXAMPLE: BINARY SEARCH
+     TREE
+As a running example to explain the proposed method
+of applying the PBT tool QuickCheck for MT, we
+consider the operations of inserting into and deleting
+from a binary search tree (BST). This example not
+trivial but is simple enough to explain the proposed
+method. Another reason for choosing BST is that the
+same approach can be used for testing more elaborate       Figure 2: MR property: Tree 1 and Tree 2 are semantically
+                                                           equivalent
+kinds of trees such as Merkle trees [22], see also our
+discussion future works in Section 6. To evaluate the
+proposed approach, we also introduce faulty variants       To test the effectiveness of the proposed method, we
+of the operations under test, insert and delete.           intentionally introduce faulty variants of insert and
+    A BST is a type of data structure for storing val-     delete and test them in a similar way. The faulty vari-
+ues such as integers in an organized way. The internal     ants are:
+nodes of BST store a key greater than all the keys in
+                                                               Fault 1 insert removes the original tree and re-
+the node’s left subtree and less than those in its right
+                                                               turns just the newly inserted value in a single
+subtree. BST are usually used for fast lookup, in-
+                                                               node.
+sert and delete of value items. Testing insert function
+which inserts a key and value in a binary search tree          Fault 2 delete does not build the tree above the
+is difficult. Using MT approach, we can change the             key being deleted. That is, it only returns the rest
+input using a new key and value and then observe the           of the tree instead.
+relationship to the original call to insert function. MT
+allows more numbers of properties to be tested. Us-        Starting with the declaration of the data type, a BST
+ing the example of trees, we can use insert with delete    for some key k and value v, is either a Lea f or a
+and test the output. Inversely, we can use delete with     Branch containing left subtree, key k, value v and the
+insert and test the output. This is true for any combi-    right subtree, respectively.
+nation of the operations under test.                           Step 1: Specify an MR property. This is the
+    One possible mistake when testing properties of        property that we wish the PBT tool to check. Before
+the insertion and deletion of BST, is that the test code   we can choose the MR, we need to pick the functions
+is the same as the implementation. Therefore, if there     that we want to test. For this example we choose in-
+is a bug in the implementation, it will also be in the     sert and delete. insert takes key k, value v and the tree
+tests which renders the tests useless. One solution to     and returns the modified tree after the insertion. The
+this problem is to get an appropriate metamorphic re-      delete function takes key k and value v and returns the
+lation to test the intended behaviour. This way we can     modified tree after the deletion.
+verify the correctness of the implementation without           Since we want to test two distinct functions (in-
+a expecting concrete output.                               sert and delete), there are, at least, two MRs that we
+    Figure 2 shows the MT of inserting keys and val-
+identify. The MRs that we want to check are the fol-           and generate the minimum failing examples for both
+lowing:                                                        of the introduced faults. However, one interesting ob-
+  • MR 1: Inserting into the tree after modifying it           servation is that fault 1 is missed by the checker when
+    with a delete operation should be the same as do-          we don’t check both MR at the same time. There-
+    ing the deleting before inserting                          fore, it is recommended to include as many MRs as
+                                                               necessary in a single test to specify properties of the
+  • MR 2: Deleting from the tree after modifying it            function under test.
+    with inserting, should be the same as doing the                One misconception of MT is that any property can
+    inserting before deleting                                  be considered as an MR, see [7]. It is true that MR
+    Table 1 shows the precise Metamorphic Relations            is a property but the inverse is not true. Therefore,
+(properties) for inserting and deleting in the context         when using PBT tools to test MR properties, we al-
+of a binary search tree. The first set shows the inser-        most always use two operations, at least, in a single
+tion of key k and value v into the tree modified by the        metamorphic test. More precisely, when using PBT
+deletion of key k0 from the original tree. The MR as-          tools to test metamorphic relations, we should either
+serts this should be equivalent to deleting k from the         change the input to same function as shown in Figure
+tree modified by insertion key k and value v into the          2 or use two distinct operations as shown in Table 1.
+same tree. The second is set of operations shows the
+deletion of key k into the tree t modified by the inser-
+tion of key k0 and value v0 . Again, the MR asserts this
+should be equivalent to doing the deletion of k first,
+                                                               6    CONCLUSIONS
+then, inserting k0 and v0 .
+                                                               In this paper, we presented a systemic method for
+    This demonstrates how effective MT can be for
+                                                               using PBT tools to automate the test generation and
+generating properties. That is, if the number of opera-
+                                                               verification of metamorphic relations. Many existing
+tions is n, the number of derived operations is O(n2 ),
+                                                               efforts for automating MT are domain-specific, i.e.
+see also [19].
+                                                               the automation of the MT steps is elaborated to work
+    Step 2: Customize the test case generator. We
+                                                               only for specific application domains such as web ser-
+use the generator defined in section 4 which generate
+                                                               vices and specific programming languages. The work
+random trees by creating a random list of keys and
+                                                               presented in this paper is more general and can be
+a random list of values and inserting them into the
+                                                               used in many different scenarios where MT is needed.
+empty tree using insert function. We also have to de-
+                                                               Its advantage is in using authenticated data structures
+fine valid function which ensures the following:
+                                                               (ADS) to solve the issue.
+  • All the keys in the left subtree is less than the key          PBT tools are generally used for testing univer-
+    at the root node                                           sal properties other than MR such as postconditions,
+  • All the keys in the right subtree is greater than the      inductive properties, and model-based properties. In
+    key at the root node                                       this paper, we have shown a method to created a spe-
+    Step 3: Customize the test case shrinker. Using            cialized test-case generator and test-case shrinker to
+the default shrink function, shrink might include in-          automate some parts of MT steps. We showed that
+valid trees. The library may shrink the test case before       The default shrinker is not ideal for testing some kinds
+reporting it. Or It may produce a valid tree with an in-       of MR as it is difficult to compose previously de-
+valid shrinking. Therefore, we must add the precon-            fined MR to create new MRs. In addition, the default
+dition discussed in 4 to ensure only valid trees partic-       shrinker may report confusing failure cases since it is
+ipates the shrinking process. This precondition holds          based on defining shrinking on datatypes which forces
+for any randomly generated test. The precondition is           the user to add additional duplicated code. However,
+just the valid function defined in Step 2.                     this workaround is not needed with our shrinker since
+    Step 4: Run the checker. The checker is just a             it does not have to be defined on the datatypes and
+function that takes any property and returns a Boolean         there would no need to encode the invariants into the
+value. We pass the MRs relations to the checker func-          shrinkers, which requires more effort and could be
+tion then the library will generate many test cases.           difficult if the scenario is more complicated. We have
+The number can be set when configuring the checker.            implemented our method using one particular PBT
+    For the correct variants of insert and delete, the         tool QuickCheck. However, our method is general and
+PBT library reports a 100 passing tests. The number            can be implemented using any other PBT tool.
+of the generated test case can also be configured to               We presented our method using the Binary search
+increase the assurance of the test. For the faulty vari-       tree example. The two operations we selected were
+ants, the tests report failing of tests after 100 test cases   insert and delete and we introduced faulty versions of
+                                 Table 1: Some MR properties for a BST insert and delete
+                         op 1       op 2                 Metamorphic properties
+                        insert     delete    insert k v (delete k’ t)= delete k’ (insert k v t)
+                        delete     insert   delete k (insert k’ v’ t) = insert k’ v’ (delete k t)
+
+
+these two operations. We showed it is recommended               [8] Tsong Yueh Chen, Fei-Ching Kuo, Robert G
+to use as many MRs as necessary to specify the oper-                Merkel, and TH Tse. Adaptive random testing:
+ations under test otherwise the test might miss some                The art of test case diversity. Journal of Systems
+subtle faults.                                                      and Software, 83(1):60–66, 2010.
+    For future work, we plan to combine the proposed            [9] Tsong Yueh Chen, Fei-Ching Kuo, Dave Towey,
+approach to our earlier works presented in [1, 2],                  and Zhi Quan Zhou. A revisit of three studies
+where we used PBT tools to test models generated                    related to random testing. Science China Infor-
+by formal methods tools such as TLA+ [17]. Another                  mation Sciences, 58(5):1–9, 2015.
+direction of the future work is to consider the human-
+                                                               [10] Tsong Yueh Chen, Pak-Lok Poon, and Xiaoyuan
+centered aspects to enhance the proposed framework,
+                                                                    Xie. Metric: Metamorphic relation identifica-
+following the ideas presented in [25, 28, 27, 26, 31].
+                                                                    tion based on the category-choice framework.
+                                                                    Journal of Systems and Software, 116:177–190,
+                                                                    2016.
+REFERENCES                                                     [11] Koen Claessen and John Hughes. QuickCheck:
+                                                                    A lightweight tool for random testing of Haskell
+ [1] Nasser Alzahrani, Maria Spichkova, and
+                                                                    programs. In Functional Programming, pages
+     Jan Olaf Blech. Spatio-temporal models for
+                                                                    268–279, 2000.
+     formal analysis and property-based testing.
+     In Federation of International Conferences                [12] Koen Claessen and Michał H Pałka. Splittable
+     on Software Technologies: Applications and                     pseudorandom number generators using cryp-
+     Foundations, pages 196–206. Springer, 2016.                    tographic hashing. ACM SIGPLAN Notices,
+                                                                    48(12):47–58, 2013.
+ [2] Nasser Alzahrani, Maria Spichkova, and
+     Jan Olaf Blech. From temporal models to                   [13] Arnaud Gotlieb and Bernard Botella. Auto-
+     property-based testing. In Evaluation of Novel                 mated metamorphic testing. In Computer Soft-
+     Approaches to Software Engineering, pages                      ware and Applications Conference, pages 34–
+     241–246. SciTePress, 2017.                                     40. IEEE, 2003.
+ [3] Andrew W Appel, Lennart Beringer, Adam                    [14] Paul Hudak, John Hughes, Simon Peyton Jones,
+     Chlipala, Benjamin C Pierce, Zhong Shao,                       and Philip Wadler. A history of haskell: being
+     Stephanie Weirich, and Steve Zdancewic. Po-                    lazy with class. In History of programming lan-
+     sition paper: the science of deep specification.               guages, pages 12–1, 2007.
+     Philos. Trans. R. Soc. A., 375(2104), 2017.               [15] John Hughes, Benjamin C Pierce, Thomas Arts,
+ [4] Matthias Brun and Dmitriy Traytel. Generic au-                 and Ulf Norell. Mysteries of dropbox: property-
+     thenticated data structures, formally. In Interac-             based testing of a distributed synchronization
+     tive Theorem Proving, 2019.                                    service. In Software Testing, Verification and
+                                                                    Validation, pages 135–145. IEEE, 2016.
+ [5] FT Chan, TY Chen, Shing Chi Cheung, MF Lau,
+     and SM Yiu. Application of metamorphic test-              [16] Daniel Jackson. Software Abstractions: logic,
+     ing in numerical analysis. In Int. Conf. on Soft-              language, and analysis. MIT press, 2012.
+     ware Engineering, 1998.                                   [17] Leslie Lamport. The temporal logic of actions.
+ [6] Tsong Yueh Chen. Metamorphic testing: A sim-                   ACM Tran. on Programming Languages and
+     ple method for alleviating the test oracle prob-               Systems, 16(3):872–923, 1994.
+     lem. In Automation of Software Test, pages 53–            [18] Leslie Lamport. Specifying systems, volume
+     54. IEEE, 2015.                                                388. Addison-Wesley Boston, 2002.
+ [7] Tsong Yueh Chen, Fei-Ching Kuo, Huai Liu,                 [19] Huai Liu, Fei-Ching Kuo, Dave Towey, and
+     Pak-Lok Poon, Dave Towey, T. H. Tse, and                       Tsong Yueh Chen. How effectively does meta-
+     Zhi Quan Zhou. Metamorphic Testing: A Re-                      morphic testing alleviate the oracle problem?
+     view of Challenges and Opportunities. ACM                      IEEE Transactions on Software Engineering,
+     Computing Surveys, 51(1):1–27, April 2018.                     40(1):4–22, 2013.
+[20] Huai Liu, Xuan Liu, and Tsong Yueh Chen. A             Evaluation of Novel Software Approaches to
+     new method for constructing metamorphic rela-          Software Engineering, pages 396–402, 2016.
+     tions. In Quality Software, pages 59–68. IEEE,    [32] Hong Zhu. Jfuzz: A tool for automated java unit
+     2012.                                                  testing based on data mutation and metamorphic
+[21] Johannes Mayer and Ralph Guderlei. An em-              testing methods. In Trustworthy Systems and
+     pirical study on the selection of good metamor-        Their Applications, pages 8–15. IEEE, 2015.
+     phic relations. In Computer Software and Appli-
+     cations Conference, volume 1, pages 475–484.
+     IEEE, 2006.
+[22] Ralph C Merkle. A digital signature based on a
+     conventional encryption function. In Theory and
+     application of cryptographic techniques, pages
+     369–378. Springer, 1987.
+[23] Andrew Miller, Michael Hicks, Jonathan Katz,
+     and Elaine Shi.     Authenticated data struc-
+     tures, generically. ACM SIGPLAN Notices,
+     49(1):411–423, 2014.
+[24] S. Segura, D. Towey, Z. Q. Zhou, and T. Y.
+     Chen.     Metamorphic testing: Testing the
+     untestable. IEEE Software, 37(3):46–53, 2020.
+[25] Maria Spichkova, Huai Liu, Mohsen Laali, and
+     Heinz W Schmidt. Human factors in software
+     reliability engineering. Workshop on Applica-
+     tions of Human Error Research to Improve Soft-
+     ware Engineering, 2015.
+[26] Maria Spichkova and Milan Simic. Human-
+     centred analysis of the dependencies within
+     sets of proofs. Procedia computer science,
+     112:2290–2298, 2017.
+[27] Maria Spichkova and Anna Zamansky. Ahr:
+     Human-centred aspects of test design. In Inter-
+     national Conference on Evaluation of Novel Ap-
+     proaches to Software Engineering, pages 111–
+     128. Springer, 2016.
+[28] Maria Spichkova and Anna Zamansky.        A
+     human-centred framework for supporting agile
+     model-based testing. In CAiSE Forum, pages
+     105–112, 2016.
+[29] Qiuming Tao, Wei Wu, Chen Zhao, and Wuwei
+     Shen. An automatic testing approach for com-
+     piler based on metamorphic testing technique.
+     In Asia Pacific Software Engineering Confer-
+     ence, pages 270–279. IEEE, 2010.
+[30] Elaine J Weyuker. On testing non-testable pro-
+     grams. The Computer Journal, 25(4):465–470,
+     1982.
+[31] Anna Zamansky, Guillermo Rodriguez-Navas,
+     Mark Adams, and Maria Spichkova. Formal
+     methods in collaborative projects. In Proceed-
+     ings of the 11th International Conference on
+

packages/opencode/specs/simulation-research/text/2022-stairway-to-verification.txt

@@ -0,0 +1,982 @@
+      Property-Based Testing: Climbing the Stairway to
+                        Verification
+                    Zilin Chen                                     Christine Rizkallah                                 Liam O’Connor
+                UNSW Sydney                                      University of Melbourne                           University of Edinburgh
+               Sydney, Australia                                   Melbourne, Australia                                Edinburgh, UK
+       zilin.chen@student.unsw.edu.au                      christine.rizkallah@unimelb.edu.au                       l.oconnor@ed.ac.uk
+
+                 Partha Susarla                                         Gerwin Klein                                    Gernot Heiser
+              Melbourne, Australia                             Proofcraft and UNSW Sydney                               UNSW Sydney
+              partha@spartha.org                                     Sydney, Australia                                 Sydney, Australia
+                                                                   kleing@unsw.edu.au                                gernot@unsw.edu.au
+
+                                                                      Gabriele Keller
+                                                                     Utrecht University
+                                                                    Utrecht, Netherlands
+                                                                      g.k.keller@uu.nl
+
+Abstract                                                                               Keywords: QuickCheck, functional programming, formal
+Property-based testing (PBT) is a powerful tool that is widely                         verification, systems programming
+available in modern programming languages. It has been                                 ACM Reference Format:
+used to reduce formal software verification effort. We demon-                          Zilin Chen, Christine Rizkallah, Liam O’Connor, Partha Susarla,
+strate how PBT can be used in conjunction with formal ver-                             Gerwin Klein, Gernot Heiser, and Gabriele Keller. 2022. Property-
+ification to incrementally gain greater assurance in code                              Based Testing: Climbing the Stairway to Verification. In Proceedings
+correctness by integrating PBT into the verification frame-                            of the 15th ACM SIGPLAN International Conference on Software
+work of Cogent—a programming language equipped with                                    Language Engineering (SLE ’22), December 06–07, 2022, Auckland,
+a certifying compiler for developing high-assurance systems                            New Zealand. ACM, New York, NY, USA, 14 pages. https://doi.org/
+components. Specifically, for PBT and formal verification to                           10.1145/3567512.3567520
+work in tandem, we structure the tests to mirror the refine-
+ment proof that we used in Cogent’s verification framework:                            1    Introduction
+The expected behaviour of the system under test is captured                            Property-based testing (PBT), in the style of QuickCheck [20],
+by a functional correctness specification, which mimics the                            is a popular testing methodology and is supported in many
+formal specification of the system, and we test the refinement                         modern programming languages [50]. In PBT, tests are spec-
+relation between the implementation and the specification.                             ified using logical properties, which are automatically exe-
+We exhibit the additional benefits that this mutualism brings                          cuted on randomly generated inputs in search of counter-
+to developers and demonstrate the techniques we used in                                examples. PBT is not only useful in finding bugs in programs,
+this style of PBT, by studying two concrete examples.                                  it has also been leveraged to reduce the effort in formal verifi-
+                                                                                       cation [15, 29, 37, 47]. Subjecting code to extensive PBT prior
+CCS Concepts: • Software and its engineering Software                                  to verification reduces the number of defects and specifica-
+testing and debugging; Functionality; Formal software                                  tion inconsistencies, thus reducing verification cost. Proof
+verification; Designing software.                                                      engineers can first test a property and only attempt to prove
+                                                                                       it after having gained reasonable confidence in its validity.
+                                                                                          In program verification, it is common practice to prove
+Permission to make digital or hard copies of all or part of this work for              the correctness of a program against a formal specification.
+personal or classroom use is granted without fee provided that copies are not
+made or distributed for profit or commercial advantage and that copies bear            The specification can be given in various forms (e.g. state
+this notice and the full citation on the first page. Copyrights for components         machines, process calculi, modal logics), depending on the
+of this work owned by others than ACM must be honored. Abstracting with                application domain. To show that the implementation con-
+credit is permitted. To copy otherwise, or republish, to post on servers or to         forms to the specification, the notion of refinement [7, 24, 52]
+redistribute to lists, requires prior specific permission and/or a fee. Request        is frequently used to establish the formal connection.
+permissions from permissions@acm.org.
+SLE ’22, December 06–07, 2022, Auckland, New Zealand
+                                                                                          In this work, we explore the combination of PBT and
+© 2022 Association for Computing Machinery.                                            refinement-based formal verification. We borrow from veri-
+ACM ISBN 978-1-4503-9919-7/22/12. . . $15.00                                           fication the functional correctness specification that is used to
+https://doi.org/10.1145/3567512.3567520                                                dictate the behaviour of the system in question, and give it
+
+
+
+                                                                                  84
+SLE ’22, December 06–07, 2022, Auckland, New Zealand               Z. Chen, C. Rizkallah, L. O’Connor, P. Susarla, G. Klein, G.Heiser, and G. Keller
+
+
+to PBT. Instead of testing logical properties about the sys-               We examine these benefits by integrating PBT into the
+tem, which is what PBT is typically designed for, we test               Cogent framework. The BilbyFs file system [3], developed
+the refinement relation between the implementation and                  in Cogent, provides an example of this effect. The entire Bil-
+the specification. Using logical properties to describe the             byFs had been formally specified but only partially verified.
+behaviour of a system has been criticised for its practicabil-          By applying PBT, we uncovered bugs in the specification and
+ity [43], especially if the full functional correctness of the          the implementation of BilbyFs. PBT has therefore already
+system is desired. Instead, high-level properties of a system           reduced the cost of verifying the remainder of the system by
+can be proved on top of its functional specification.                   uncovering mistakes early on.
+   We introduce PBT to our development cycle, parallel to                  This work builds on top of a preliminary investigation [18].
+the refinement-based verification framework. Specifically,              To summarise, we make the following contributions:
+we formulate the refinement between the implementation                       • We demonstrate how to integrate PBT into a refine-
+and the functional specification as the property to be tested,                 ment verification framework by using Cogent as the
+which is an under-explored application of PBT. Employing                       target platform. Unlike previous use of PBT, our test
+PBT in the formal verification context brings additional ben-                  specification is defined in terms of refinement proper-
+efits beyond detecting bugs in the implementation of the                       ties (Section 3).
+systems.                                                                     • We argue why PBT is suitable to be employed in paral-
+   In contrast to the high-effort all-or-nothing of a full func-               lel with formal verification, and explain the important
+tional correctness proof, PBT provides a continuum ranging                     role that PBT plays in the design and implementation
+from no assurance (no tests), to some assurance (good test                     of the systems (Section 4).
+coverage), to better assurance (some properties proved, some                 • We provide two concrete examples from the testing of
+tested), and all the way to high assurance (all properties                     components of the BilbyFs file system to demonstrate
+proved). This allows users to make trade-offs between cost                     techniques that we used for specifying refinement re-
+and assurance according to the criticality of a component.                     lations, modularising the tests, using mocks, handling
+   Tests are more immune to program evolution than formal                      non-determinism, and efficiently generating test data
+proofs. A proof may require significant changes whenever                       (Section 5 and Section 6).
+the code changes, even in scenarios where the specifica-                     • We discuss the engineering implications of our ap-
+tion remains the same (e.g. optimisation). On the contrary,                    proach and lessons learnt and proposals resulting from
+PBT only requires developers’ input when the specification                     them (Section 7).
+changes. Therefore, PBT can provide quicker feedback on
+the correctness of the change, reducing code maintenance
+                                                                        2     Background
+cost. Furthermore, all proofs depend on assumptions such as
+the correctness of the hardware or the external software in-            2.1 Cogent
+volved. Some of these assumptions can be tested to increase             From the experience of formally verifying the seL4 microker-
+the correctness of the overall system.                                  nel, Klein et al. [41] have observed that the proofs connecting
+   It is usually challenging to verify software that was not            the high-level specifications with the low-level C systems
+designed for verification, as the code has to be structured in          code are time consuming and tedious to develop, but are
+a modular fashion, around clearly stated correctness prop-              not particularly involved. As such they are good candidates
+erties. This means that it is vital to have an effective means          for automation. The Cogent verification framework was
+for developers to express their design requirements in order            conceived to partly automate these proofs.
+to experiment with and evaluate their design, and to have a                Cogent [55, 56, 58] is a purely functional language
+good set of design guidelines [14] for them to write programs           that was developed to reduce the cost of developing high-
+that can be readily specified and verified.                             assurance systems components. Similar to the Rust lan-
+   In large-scale software verification projects, such as               guage [39], Cogent is equipped with a uniqueness type sys-
+seL4 [41], systems developers and verification experts are              tem [8, 11, 25, 66] that ensures memory safety, easing the bur-
+typically from two separate teams. We posit that PBT specifi-           den of verification. Cogent’s type system allows imperative-
+cations can be used to enhance the communication between                style destructive updates, while retaining a purely functional
+these two groups. While the properties are similar to formal            semantics. The type system eliminates the need for a garbage
+specifications, they represent tests and, as such, feel more            collector, making the language more suitable for systems
+familiar to software engineers than abstract proof require-             code and also easier to verify.
+ments. Since PBT gives almost immediate benefit to software                The uniqueness types come at a cost: it is impossible in
+engineers, there is an incentive for them to design their code          Cogent to implement data structures and functions which,
+such that these properties are meaningful and easy to ex-               even temporarily, rely on sharing. Instead, they have to be
+press, thereby structuring their code for formal specification,         implemented in C, verified separately, and imported as ab-
+making it amenable to verification.                                     stract types and abstract functions through a foreign function
+
+
+
+                                                                   85
+Property-Based Testing: Climbing the Stairway to Verification                                         SLE ’22, December 06–07, 2022, Auckland, New Zealand
+
+
+          manual       legend                                                         1 type H
+                                      Functional specification (Isabelle/HOL)         2 type Bag = { count : U32, sum : U32 }
+          generated
+               proof                                                                  3 newBag : H → <Success (Bag, H) | Failure H >
+                                                                                      4 freeBag : (H, Bag) → H
+
+
+
+
+                                               re nes
+                                                                                      5
+        COGENT
+        source                                                                        6 addToBag : (U32, Bag) → Bag
+                          generates   Shallow embedding of                            7 addToBag (x, b { count = c, sum = s })
+
+
+
+
+                                                                       re nes
+                                      COGENT in Isabelle/HOL
+                                                                                      8    = b { count = c + 1, sum = s + x }
+         COGENT                                                                       9
+
+
+                                               re nes
+                          generates
+        compiler                                                                     10 averageBag : Bag! → <Success U32 | EmptyBag >
+                                                                                     11 averageBag b = if b.count == 0 then EmptyBag
+                          generates                                                  12                  else Success (b.sum / b.count)
+                                                 C                 C library
+                                                                                     13
+   fi
+                                                                                     14 type List a
+                                                                                     15 reduce : ∀ (a, b). ((List a)!, (a!,b) → b, b) → b
+Figure 1. An overview of Cogent’s verification framework                             16
+                                                                                     17 average : (H, (List U32)!) → (H, U32)
+                                                                                     18 average (h, ls) = newBag h
+
+interface (FFI) that requires the uniqueness constraints to                          19    | Success (b, h') →
+                                                                                     20        let b' = reduce (ls, addToBag, b)
+be satisfied at the interface level. Besides, Cogent does not                        21         in averageBag b' !b'
+natively support loops or recursion; they also need to be                            22            | Success n → (freeBag (h', b'), n)
+implemented in C. We refer to this collection of C code as                           23            | EmptyBag → (freeBag (h', b'), 0)
+the C library.                                                                       24    | Failure h' → (h',0)
+
+   Cogent’s certifying compiler [58, 59], presented in Fig-
+ure 1, generates C code, a shallow embedding of the Cogent                                 Figure 2. A simple example of a Cogent program
+code in the interactive theorem prover Isabelle/HOL [54],
+and a proof connecting the two. The generated proof ensures
+that the C code correctly refines the Isabelle/HOL shallow                           similar to Haskell’s IO monad. The newBag function returns a
+embedding. As such any correctness properties proved about                           variant (or sum) type to indicate the possibility of allocation
+the shallow embedding also hold for the C code. Automating                           failure. The addToBag function, which adds a new data point
+the refinement proof from Cogent to C drastically reduced                            to the bag, is defined on lines 6–8. The averageBag function
+the verification effort required compared to directly verifying                      (lines 10–12) returns, if possible, the average of the numbers
+C code [4].                                                                          added to the Bag. The input type Bag! indicates that the input
+   While the Cogent compiler generates C refinement proofs                           is a read-only, non-unique view of a Bag, which is created
+automatically, it cannot prove full functional correctness,                          on line 21 using the ! notation. Lastly, lines 17–24 define the
+which is specified manually by the developer in a functional                         overall average function, which uses the Bag to compute the
+correctness specification in Isabelle/HOL. This leaves two                           average of the elements of a List.
+steps that require manual verification (the two solid arrows
+in Figure 1): (1) verifying that the purely functional shallow                       2.2    Property-Based Testing and QuickCheck
+embedding of the Cogent code refines the overall functional                          PBT is a quick and effective method for detecting bugs and
+correctness specification, and (2) verifying that each C ADT                         finding inconsistencies in specifications [38]. Similar to for-
+refines its functional specification. The former, albeit manual,                     mal verification, PBT uses logical predicates to specify the
+is eased by virtue of equational reasoning. The latter proof,                        desired behaviour of functions, by defining the allowed rela-
+which is directly on the C level, is more involved. However,                         tions between inputs and outputs of the functions. It evalu-
+the C library is to be shared across multiple systems and                            ates the properties on a large set of automatically generated
+hence the effort is amortised over time.                                             input values in search of counter-examples.
+   Figure 2 provides an example of a Cogent program. Given                              While PBT is effective, it is not universally applicable—in
+an abstract List datatype with a reduce function (line 15) that                      practice, it is often hard to describe the full behaviour of
+aggregates the List content using a provided aggregation                             a system solely in terms of logical properties [43]. In the
+function and identity element, the function average (line 17)                        context of formal verification, however, using a functional
+computes the average of a list of 32-bit unsigned integers. It                       specification to describe the behaviour of a systems is very
+accomplishes this by storing the running total and count in a                        common. Thus proof engineers can first run extensive tests
+heap-allocated data structure, called a Bag, defined on line 2,                      on the conjectures before attempting any proof development.
+with external allocation and free functions on lines 3 and 4.                        This technique is not new and has witnessed great success
+Because Cogent is a purely functional language, intrinsically                        in the verification community [10].
+impure functions, like the memory (de-)allocation functions,                            QuickCheck [20] is a combinator library in Haskell for
+need to have the heap H threaded through them. This is                               PBT. While the QuickCheck functionality is now available
+
+
+
+                                                                                86
+SLE ’22, December 06–07, 2022, Auckland, New Zealand               Z. Chen, C. Rizkallah, L. O’Connor, P. Susarla, G. Klein, G.Heiser, and G. Keller
+
+
+in many programming languages [50] and theorem provers                             manual      legend
+                                                                                                                Executable specification (Haskell)
+[15, 29, 47], we interface Cogent to the Haskell QuickCheck                        generated
+library, as it is mature, feature-rich and integrates well with                     testing
+
+
+
+
+                                                                                                                        re nes
+Cogent and C.                                                                                                                    (1)
+                                                                                 COGENT
+                                                                                 source
+
+
+
+
+                                                                                                                                       re nes
+                                                                                                  generates   Shallow embedding of
+                                                                                                                                                (3)
+
+
+
+
+                                                                                                                                                         re nes
+2.3   Data Refinement                                                                                           COGENT in Haskell                                 (2)
+
+Prior verification work in Cogent, e.g. of BilbyFs [3], con-                      COGENT
+
+
+
+
+                                                                                                                        re nes
+nects the functional specification to the Cogent implemen-                       compiler                                        (4)
+tation, and the Cogent implementation to the compiled C
+                                                                                                  generates
+code [59] by proving refinement relations. The notation of                                                                C                           C library
+refinement is also central to our testing framework, in which               fi
+
+
+
+they are expressed as QuickCheck properties. We use a text-
+                                                                        Figure 3. An overview of the Cogent QuickCheck frame-
+book definition of refinement [24]. Informally, a program 𝐶
+                                                                        work
+is a refinement of a program 𝐴 if every possible behaviour in
+the model of 𝐶 is observable in that of 𝐴.
+   In an imperative setting, a simple model for both the ab-            statement merely requires the single concrete result to cor-
+stract specification and the concrete implementation would              respond to one of the possible abstract results:
+be relations on states, describing every possible behaviour of                        𝑅𝑋 𝑖𝑎 𝑖𝑐 =⇒ ∃𝑜 𝑎 ∈ abs 𝑖𝑎 . 𝑅𝑌 𝑜 𝑎 (conc 𝑖𝑐 )
+the program as the manipulation of some global state. This
+means that if we prove a property about every execution for             Defining the notation
+our abstract specification, we know that the property holds                                                       def
+                                                                                                  corres 𝑅 𝑎 𝑐 = ∃𝑜 ∈ 𝑎. 𝑅 𝑜 𝑐
+for all executions of our concrete implementation.
+   This state-based model for specifying the behaviours of              the refinement statement can be formulated as:
+systems is a very common paradigm in the world of model-                                𝑅𝑋 𝑖𝑎 𝑖𝑐 =⇒ corres 𝑅𝑌 (abs 𝑖𝑎 ) (conc 𝑖𝑐 )
+based testing (MBT) [33, 43, 64]. However, as mentioned in              Theorems that capture the functional correctness of Cogent
+Section 1, Cogent’s purely functional semantics provides                systems typically have this corres format. We therefore aim to
+a simple formal model of a program’s behaviour; specifi-                encode these as machine-testable properties in QuickCheck.
+cally, it enables reasoning about programs using equational
+principles. The equational semantics is fortunately widely              3        The Cogent QuickCheck Framework
+available in PBT libraries, including QuickCheck.
+                                                                        The integration of PBT into the Cogent framework mirrors
+   Since Cogent is a purely functional, deterministic, total
+                                                                        the verification tasks, as shown in Figure 3. The developer
+language, there is no global state, and all functions are mod-
+                                                                        manually writes a Haskell executable specification, which
+elled as plain mathematical functions. In such a scenario,
+                                                                        plays a similar role to the Isabelle/HOL functional correct-
+the only state involved consists of the inputs and outputs to
+                                                                        ness specification. The compiler now generates a Haskell
+the function, simplifying the refinement statement. Given
+                                                                        shallow embedding of the Cogent code for PBT. Although
+an abstract function abs :: 𝑋𝑎 → 𝑌𝑎 , and a concrete Cogent
+                                                                        not formally connected, the Haskell and Isabelle/HOL em-
+function conc :: 𝑋𝑐 → 𝑌𝑐 , then, assuming the existence of re-
+                                                                        beddings are very similar.
+finement relations 𝑅𝑋 and 𝑅𝑌 , we can express the statement
+                                                                           The framework supports testing the C implementation of
+that conc refines abs as:
+                                                                        ADTs against the Haskell executable specification, shown as
+                                                                        arrow (2) in Figure 3; Section 5 provides an example. It fur-
+              𝑅𝑋 𝑖𝑎 𝑖𝑐 =⇒ 𝑅𝑌 (abs 𝑖𝑎 ) (conc 𝑖𝑐 )                       thermore supports testing the Cogent program along with
+                                                                        ADTs that the Cogent program uses, against the Haskell exe-
+This, however, places unnecessary constraints on our ab-                cutable specification. This is depicted as arrow (1) in Figure 3
+stract specification. While Cogent is deterministic and total,          (the included ADTs are not shown in the figure); Section 6
+our abstract specification need not be. In fact, it is often de-        provides a case-study. The included ADTs can either be real
+sirable to allow non-determinism to reduce the complexity               C implementations (via Haskell’s C FFI) or Haskell mocks,
+of the abstract specification. In the context of testing, we are        depending on whether the ADTs are also considered the
+required to restrain the degree of non-determinism in the               system under test.
+specification, for the sake of efficient execution of the test             The refinement relation between the C code and the
+script. This, however, does not preclude us from having a               Haskell embedding of the Cogent program (arrow (4)) can
+non-deterministic specification.                                        also be tested, although it does not concern us as much, since
+   We model non-determinism by allowing abstract func-                  it is certified by the automatic proof. One scenario where
+tions to return a set of possible results. Then, our refinement         this test can be beneficial is during the development of the
+
+
+
+                                                                   87
+Property-Based Testing: Climbing the Stairway to Verification                          SLE ’22, December 06–07, 2022, Auckland, New Zealand
+
+
+Cogent compiler, before the automatic refinement proof                  when the Isabelle/HOL specification is developed prior to the
+pipeline is fully restored.                                             Haskell executable specification. This however is not always
+   The complete compiled executable, depicted in Figure 3               the case, and in fact in the workflow that we proposed, the
+as the grey dotted box at the bottom, can be tested against             Haskell specification is used to guide the formulation of the
+the executable specification in theory, as indicated by arrow           Isabelle/HOL one.
+(3). However, we typically do not perform this test using
+the QuickCheck framework in Cogent, as the gap between
+their state spaces is usually too large to handle effectively.
+                                                                        4   PBT and Systems Design Go
+The final system can normally be deployed in the production                 Hand-in-Hand
+environment and be tested against third-party test suites               We argue that the employment of PBT and the design of the
+for their specific application domain. In the context of the            systems are interlinked with each other: appropriate systems
+BilbyFs, for example, there exist tools such as fstest [9].             design assures the effectiveness of testing, and the use of
+   On the formal verification end, the Isabelle/HOL func-               PBT encourages the programmers to design their systems in
+tional correctness specification may be highly non-                     a fashion that is amenable to formal verification.
+deterministic in order to succinctly characterise external fac-            As a purely functional language, Cogent is well suited
+tors such as the elapsed time of an operation, out-of-memory            for PBT and verification: the result of a executing a function
+errors or hardware errors. The Haskell specification must               only depends on the input, with no hidden state. Instead, a
+also be capable of modelling non-determinism, but in a more             system state must be explicitly threaded through an impure
+controlled manner, as the specification must be executable.             function. If the function is not designed appropriately, it is
+Simulating a non-deterministic model can be exponential                 possible that a PBT test suite hardly yields counter-examples.
+in time and space. To allow modelling a minimal amount                  Systems code often involves large global states. However, a
+of non-determinism in the Haskell specification, the tester             function typically only accesses a small portion of the state.
+has to ensure that the search domain is finite and reasonably           If the entire state is passed in, any random variations to the
+small by carefully examining the needed quantifiers in the              parts of the state that are irrelevant to the function will have
+specification.                                                          no effect on the execution of the function. In this case a large
+   For example, in the Isabelle/HOL abstract specification              proportion of the randomly generated test cases in PBT will
+of BilbyFs, the afs_get_current_time function is defined as             be wasted, rendering the test suite ineffective.
+follows:                                                                   In practice, this means that to be suitable for PBT, the
+  definition                                                            functions must be designed to keep the inputs minimal and
+    afs_get_current_time :: afs_state ⇒ (afs_state ×                    relevant, which is unlikely when naïvely translating exist-
+       TimeT) cogent_monad                                              ing C code with global state into Cogent. While this may
+  where
+    afs_get_current_time afs ≡ do
+                                                                        seem like a high price to pay, a verification-friendly design
+       time ′ ← return (a_current_time afs);                            has the same requirement for modularity and compartmen-
+       time ← select {x. x ≥ time ′ };                                  talisation [3, 5]. Thus PBT imposes few restrictions beyond
+       return (afsL a_current_time := time M, time ′ )                  existing requirements of verification, and instead helps guide
+     od                                                                 the design.
+It non-deterministically picks a time, which is no earlier than            In the context of Cogent, developers typically do not have
+the time stored in the file system state afs. This abstract             a formal specification to begin with when implementing a
+specification is not suitable for testing, due to the infinitely        new system. Systems programmers, together with verifica-
+large set of values. In contrast, on line 13 of Figure 5b, the          tion experts, not only need to ensure that they are imple-
+specification non-deterministically chooses an error code               menting the systems right, but also to ensure that they are
+from a small set of {eIO, eNoMem, eInval, eBadF, eNoEnt}.               implementing the right systems. PBT helps them structure
+This moderate state explosion can potentially be handled by             the implementation, as well as the specification. Having
+the testing framework, depending on the context in which                good design decisions, such as keeping the states threaded
+the afs_readpage function is applied.                                   through small and relevant as we mentioned above, is dou-
+   The Haskell executable specification is thus often strictly          bly rewarding: it keeps the refinement relation between the
+less abstract than the Isabelle/HOL functional specification.           concrete and the abstract states simple, both in PBT and in
+Although it is not necessary to formally connect the two,               verification.
+as the gap only affects the quality of the tests, ideally we               Expressing design requirements for verification is an on-
+would want one specification to be generated from the                   going challenge, and we observed in the past that when veri-
+other one. Automatic refinement mechanisms that allow                   fying real-world systems, it is difficult for proof engineers to
+verified generation of Haskell from Isabelle/HOL have been              communicate these requirements effectively to the software
+explored [44, 45]. Generating the Haskell specification from            engineers. The seL4 project [41] overcame this problem by
+the Isabelle/HOL abstract specification is undoubtedly handy,           using an executable Haskell specification of the system as
+
+
+
+                                                                   88
+SLE ’22, December 06–07, 2022, Auckland, New Zealand             Z. Chen, C. Rizkallah, L. O’Connor, P. Susarla, G. Klein, G.Heiser, and G. Keller
+
+
+    prop_corres_wordarray_set_u8 :: Property                          the polymorphic wordarray_set function, whose refinement
+    prop_corres_wordarray_set_u8 = monadicIO $                        statement is given in Figure 4. It can be read roughly as:
+      forAllM gen_wordarray_set_u8_arg $ \args → run $ do
+                                                                      for any type-correct concrete input ic and its abstraction
+        let ia = mk_hs_wordarray_set_u8_arg args
+            oa = uncurry4 hs_wordarray_set ia
+                                                                      ia, check that the result (i.e. oc) of applying the concrete
+        ic ← mk_c_wordarray_set_u8_arg args                           function cogent_wordarray_set_u8 and that of the abstract
+        oc ← cogent_wordarray_set_u8 ic                               function hs_wordarray_set (i.e. oa) are related via the refine-
+        corresM' rel_wordarray_u8 oa oc                               ment relation rel_wordarray_u8. The corresM' function is a
+                                                                      monadic variant of our corres notation for situations where
+Figure 4. The refinement statement for wordarray_set (de-             the specification is deterministic.
+allocation is omitted for simplicity)                                     Although the corres predicate contains an existential quan-
+                                                                      tifier for the result of the non-deterministic abstract specifi-
+                                                                      cation, our implementation does not require QuickCheck to
+an interface between these two groups [40]. We posit that             guess the quantified value from a set of all possible values.
+QuickCheck properties is a highly suitable language for com-          Instead, our test driver enumerates over all possible output
+municating design requirements. They readily translate to             values of the abstract function to find the existentially quan-
+formal specifications, but are expressed in a programming             tified value. In Section 6, we show how to restrict the size of
+language, and thus familiar to software developers. As they           the abstract function’s codomain for the enumeration to be
+lead to effective test generation, software developers get im-        tractable.
+mediate benefit from using these specifications to structure              For the WordArray type, we relate the abstract input data
+their code to maximize their use, which consequently makes            and the concrete input data in the following way: we ran-
+the code easier to verify.                                            domly generate test data on a middle-ground type, and
+                                                                      then use two thin wrappers mk_hs_wordarray_set_u8_arg
+5     Example: The WordArray Library                                  and mk_c_wordarray_set_u8_arg to convert the generated
+We apply the Cogent QuickCheck framework to the                       data to the types expected by the abstract and the concrete
+WordArray library, which implements common functions ma-              functions. The test data generation is not very involved, be-
+nipulating arrays of machine words and is shared by all               cause the correspondence between the two types is straight-
+of our systems implementations. Most of these WordArray               forward, and the C type is not very convoluted in its under-
+functions are implemented in C, and are invoked via the FFI           lying representation, in particular it does not heavily use
+mechanism available in Cogent.                                        pointers.
+   We want to test whether each function observes the refine-             Although in the refinement statement, the refinement re-
+ment property from Section 2.3. For example, the behaviour            lation between the input data is expressed as a predicate, this
+of the ADT function wordarray_set (similar to memset in C)—           is usually not the way to implement the test driver. Checking
+which fills the first n elements starting at a certain index          a predicate is often straightforward, but it requires two sets
+frm into an array arr with a constant value a—is manually             of random data generators and forces the two generators to
+specified in Haskell as follows:                                      be coupled. Otherwise the correspondence predicate is likely
+    -- Haskell spec.                                                  to reject the vast majority of the generated data, rendering
+    type WordArray a = [a]                                            the test very inefficient (see Section 7.4).
+    hs_wordarray_set :: WordArray a → Word32 → Word32                     Broadly speaking, it is more convenient to relate the input
+                      → a → WordArray a                               data if we implement the refinement relation as an abstrac-
+    hs_wordarray_set arr frm n a =
+      let len = length arr in                                         tion function, computing the abstract data according to its
+       if | frm > len = arr                                           concrete counterpart. This is contrary to model-based testing
+          | frm + n > len                                             approaches [64], in which the test cases are derived from the
+          = take frm arr ++ replicate (len - frm) a                   more abstract model. We use abstraction functions because
+          | otherwise
+          = take frm arr ++ replicate n a ++
+                                                                      typically the concrete state contains more data than its ab-
+            drop (frm + n) arr                                        stract counterpart. In order to derive a concrete input from
+                                                                      the randomly generated abstract input, more data need to be
+In Cogent, the function is defined as an abstract function,
+                                                                      created and this calls for another set of random data genera-
+whose definition is given in C. The Cogent function inter-
+                                                                      tors. On the other hand, when we generate a concrete input
+face looks like:
+                                                                      and abstract it, it only requires a lossy abstraction function.
+    type WordArray a                                                      In the case of WordArray, we compute the abstract in-
+    wordarray_set : (WordArray a, U32, U32, a) → WordArray a
+                                                                      put data from the concrete one, relating them by construc-
+While Cogent and Haskell both support polymorphism, C                 tion. However, not all values of a C type are valid inputs:
+does not, and QuickCheck cannot perform genuine polymor-              for instance, a null pointer does not correspond to a valid
+phic testing [12]. In this example, we test the U8 instance of
+
+
+
+                                                                 89
+Property-Based Testing: Climbing the Stairway to Verification                          SLE ’22, December 06–07, 2022, Auckland, New Zealand
+
+
+WordArray. To exclude invalid input data, we manually im-               memcpy in C), the old implementation implicitly assumed that
+plement a test data generator gen_wordarray_set_u8_args                 the index into the source array was always within bounds.
+which generates values that are isomorphic to valid con-                This precondition was satisfied by our file system implemen-
+crete inputs only, and we convert them to Haskell inputs and            tations, but it was unspecified. In fact, the wordarray_copy
+C inputs using functions mk_hs_wordarray_set_u8_arg and                 function, as part of a generic library, should not carry this
+mk_c_wordarray_set_u8_arg respectively.                                 implicit precondition. Otherwise it may introduce bugs to
+   The refinement statement as shown in Figure 4 is largely             other customers of this library function but do not perform
+boilerplate code. To generate this code, we designed a small            the check.
+domain-specific language (DSL), whose prototype has been                   PBT also helped us uncover problems in the Isabelle/HOL
+implemented [27]. In this DSL, programmers can specify the              ADT specifications, which had overly specific assumptions
+function names, the definition of the abstraction function for          about inputs. While these assumptions are valid for the func-
+the inputs and the refinement relation between the outputs,             tions we verified, they do not hold in general. Thus, the
+and other properties about the refinement statement, such               specifications we had written did not represent a general
+as the determinism of the abstract function and whether                 purpose specification of the function.
+the concrete function needs to operate under the IO monad.                 The WordArray library in Cogent was initially axiomatised
+The DSL is written in JSON format, which can be readily                 in the verification of the file systems [4], and then tested
+parsed using third-party libraries such as aeson [57]. A piece          using the PBT framework, before they were finally formally
+of sample code is give below:                                           verified [19]. This is an example of how the developers can
+                                                                        progressively increase their confidence in the correctness
+1 {
+2     "name" : "wordarray_set_u8",                                      of the code by upgrading PBT to formal verification in a
+3     "monad" : true,                                                   modular fashion.
+4     "nondet": false,
+5     "absf" : ... // the abstraction function
+6     "rrel" : ... // ref. rel. between outputs
+                                                                        6     Example: A Top-Level File System
+7 }                                                                           Operation
+Haskell program texts can be embedded in the JSON struc-                BilbyFs [3] is a flash file system that was designed from
+ture as the values of the "absf" and "rrel" attributes (lines           scratch, focusing on modularity and verifiability; it has 19
+5 and 6). This allows programmers to either call a Haskell              top-level file system operations. Two functions have been
+function defined elsewhere, or directly write the definition            previously verified in Isabelle/HOL to demonstrate how Co-
+in-place. The lens [42] style of code is particularly suitable          gent facilitates equational reasoning. fsop_sync, a top-level
+for accessing and relating parts of deeply nested algebraic             function, consists of about 300 lines of Cogent code and
+datatypes.                                                              took approximately 3.75 person months to verify with 5700
+   In the refinement statement, the Cogent-compiled C code              lines of proof. The other function, iget, directly called by the
+can be called from Haskell using its C FFI facility. The Haskell        top-level fsop_lookup function, consists of approximately
+representation of C types, marshalling functions, and foreign           200 lines of code, and took about one person month to verify
+function calls are generated by the Cogent compiler and                 with 1800 lines of proof.
+are further compiled by FFI tools such as hsc2hs [32] and                  We conducted PBT on one of BilbyFs’s top-level func-
+c2hs [16].                                                              tion fsop_readpage [17], which had previously been formally
+   Running a small number of randomly generated tests                   specified in Isabelle/HOL but not yet verified. Figure 5 shows
+(by default 100 but this can be customised) by passing                  the Isabelle/HOL specification as well as the manually writ-
+prop_corres_wordarray_set_u8 to the quickCheck function,                ten Haskell executable specification; they are very similar in
+we get:                                                                 this case. Therefore, testing gives us reasonably high assur-
+*WordArray> quickCheck prop_corres_wordarray_set_u8                     ance of the implementation with respect to the Isabelle/HOL
++++ OK, passed 100 tests.                                               specification. As discussed in Section 3 , this is not always
+We have specified most of the ADT functions for WordArrays              the case, making it occasionally more difficult to connect
+and tested them [17]. We found bugs in two C functions,                 the two specifications, and sometimes requiring additional
+which had not been uncovered by our earlier test suites nor             manual reasoning.
+the file systems built with them. The bugs went undetected
+as they involved invalid inputs and corner cases which were             6.1   The Haskell Executable Specification
+handled by the callers, whereas the Haskell specification in            In a nutshell, as shown in the Haskell specification in Fig-
+our QuickCheck framework does not preclude these input                  ure 5a, the function hs_fsop_readpage fetches a designated
+values.                                                                 data block of a specific file to the buffer. The argument afs is
+   For example, for the wordarray_copy function that copies a           a map from inode numbers to files; each file is represented as
+number of bytes from one memory area to another (similar to             a list of blocks of data. The hs_fsop_readpage function looks
+
+
+
+                                                                   90
+SLE ’22, December 06–07, 2022, Auckland, New Zealand                Z. Chen, C. Rizkallah, L. O’Connor, P. Susarla, G. Klein, G.Heiser, and G. Keller
+
+
+ 1 hs_fsop_readpage :: AfsState
+ 2                    → VfsInode
+ 3                    → OSPageOffset
+ 4                    → WordArray U8
+ 5                    → NonDet (Either ErrCode (WordArray U8))
+ 6 hs_fsop_readpage afs vnode n buf =
+ 7    let size = vfs_inode_get_size vnode :: U64
+ 8        limit = size `shiftR` bilbyFsBlockShift
+ 9     in if | n > limit → return $ Left eNoEnt
+10           | n == limit && (size `mod` bilbyFsBlockSize == 0) →
+11               return $ Right buf
+12           | otherwise → return (Right $ fromJust (M.lookup (vfs_inode_get_ino inode) afs) !! n) <|>
+13                          (Left <$> [eIO, eNoMem, eInval, eBadF, eNoEnt])
+
+                                                  (a) The Haskell executable specification
+
+ 1 definition
+ 2     afs_readpage :: afs_state
+ 3                  ⇒ vnode
+ 4                  ⇒ U64
+ 5                  ⇒ U8 WordArray
+ 6                  ⇒ (U8 WordArray × (unit, ErrCode) R) cogent_monad
+ 7   where
+ 8     afs_readpage afs vnode n buf ≡
+ 9      if n > (v_size vnode >> unat bilbyFsBlockShift) then
+10       return (WordArrayT.make (replicate (unat bilbyFsBlockSize) 0), Error eNoEnt)
+11      else if (n = (v_size vnode >> unat bilbyFsBlockShift)) ∧ ((v_size vnode) mod (ucast bilbyFsBlockSize) = 0)
+12           then return (buf, Success ())
+13           else do err ← {eIO, eNoMem, eInval, eBadF, eNoEnt};
+14                   return (WordArray.make (pad_block ((i_data (the $ updated_afs afs (v_ino vnode))) ! unat n)
+          bilbyFsBlockSize), Success ()) ⊓
+15                   return (buf, Error err)
+16                od
+
+                                                (b) The Isabelle/HOL functional specification
+
+                                 Figure 5. Functional specifications of the fsop_readpage function
+
+up a file, whose inode number is given by vnode, in the map               here is essentially a finite set containing all allowed be-
+afs, and copies the n-th block of the file to buf. It returns             haviours. This monad is commonly used in proving refine-
+non-deterministically an updated buffer or an error code.                 ment (e.g. [3, 22]). The alternative operator (<|>) acts as a
+   As a first step, hs_fsop_readpage calculates the number of             non-deterministic choice, admitting the behaviour of either
+blocks that the wanted file occupies. If the block in question            of its operands by taking the union of their behaviours.
+is out of bounds (n > limit), the function returns a no-entry
+error eNoEnt. If the file size is a multiple of the block size,
+n points to the last block in the wanted file, and the last               6.2   Mock Implementations
+block is empty (because the file data ends at the prior block
+                                                                          It is not always feasible to test systems code in its exact pro-
+boundary), then the function returns the original buffer as
+                                                                          duction environment [53]. For instance, the fsop_readpage
+there is no data to read. Otherwise, hs_fsop_readpage reads
+                                                                          example has many low-level functions which call into the
+the block by looking up the inode number in the map (see
+                                                                          operating system’s kernel, and it is currently not feasible to
+Figure 6 for a pictorial example).
+                                                                          run QuickCheck tests in kernel mode. Instead of testing the
+   This, however, is not the only possible correct behaviour.
+                                                                          monolithic object file obtained from compiling the C code,
+As the implementation has to access buffers and read from
+                                                                          we mock up parts of the code in Haskell. A mock abstracts
+the physical medium, this may fail, in which case it should
+                                                                          from low-level kernel calls and can be thought of as a black
+throw an error. We specify this as a non-deterministic be-
+                                                                          box, which provides to its caller the same observable effects
+haviour. The specification states that the function can read
+                                                                          as the actual implementation.
+a block or it can give one of the following five errors: eIO,
+                                                                              Mocks can also be used as substitutes for unimplemented
+eNoMem, eInval, eBadF, or eNoEnt. The NonDet monad used
+                                                                          functions, enabling systems developers to test functionality
+                                                                          before they have a full system implementation. The use of
+
+
+
+                                                                     91
+Property-Based Testing: Climbing the Stairway to Verification                           SLE ’22, December 06–07, 2022, Auckland, New Zealand
+
+
+                            data                                      the mock to the specific use case of fsop_readpage. The
+                                                                      mock ostore_read function can be modelled as a simple map
+                                                           (A)        lookup: given a map OstoreState and a key of type ObjId, it
+                                                                      returns the corresponding object or an error. The relevant
+           block 0      block 1      block 2     block 3              Haskell definitions are given as follows (the use of the Oracle
+                                                                      can be ignored for now; it will be explained shortly):
+                                                           (B)
+                                                                         type OstoreState = Map ObjId Obj
+                                                                         data Res a e = Success a | Error e
+                          data
+                                                                         ostore_read :: Oracle
+Figure 6. An example of the read_page algorithm. In case                    → (OstoreState, ObjId) → Res Obj ErrCode
+(A), limit = 3. When n = 0, 1, 2 we just read. When n = 3,               ostore_read orc (ostore_st, oid) =
+                                                                           if orc == 0 then
+because the size of the data is not perfectly aligned at the                 case M.lookup oid ostore_st of
+end, we still read. When n ≥ 4, we return the no-entry error.                  Nothing → Error eNoEnt
+In case (B), limit = 3. When n = 3, that’s the special case.                   Just obj → Success obj
+We return the old buffer unmodified.                                       else Error orc
+
+
+                                                                      6.3    Oracles and Non-determinism
+mocks restricts the scope of debugging to a small number of
+functions, reducing the effort required to locate bugs.               The Cogent implementation of ostore_read interacts with
+   The Cogent implementation of fsop_readpage calls a                 the physical media and kernel data structures, therefore its
+read_block function to fetch one block of file data, which in         behaviour is dependent on that of the underlying systems
+turn retrieves the data with the function ostore_read. The            and hardware. However, as we have introduced in Section 2.1,
+read_block function, which retrieves the file data from the           Cogent is a total, purely functional language, meaning that
+physical medium, is conceptually simple. However it is com-           all functions in Cogent have to be deterministic. This non-
+plicated in its internal implementation, which involves a             determinism, therefore, has to be modelled by threading a
+red-black tree lookup to locate the address, several layers of        global state SysState through the impure functions, similar
+caching, and thorough error-handling.                                 to the H type in Figure 2.
+   Because read_block relies on kernel mode access to caches             In the Haskell mock implementation of ostore_read, the
+and complex data structures, it is a good candidate for a             non-determinism can be modelled in a different manner for
+mock implementation. In addition to the ostore_read func-             simplicity. When testing this function independently, we
+tion, we also substitute some kernel ADT functions for                emulate the non-determinism by adding an additional oracle
+mock implementations. For WordArray functions invoked                 input orc.1 This oracle can be deemed to be the source of all
+by fsop_readpage, we use the Haskell models described in              non-determinism. A similar oracle is passed to our mocks of
+Section 5 as mocks.                                                   many WordArray functions, such as wordarray_create, which
+   The implementation of a mock is simplified by the fact             allocates memory and creates a fresh array.
+that it does not need to provide the full functionality of the           We have seen how oracles can be used in mock implemen-
+operation, as long as the callers in the specific test cases          tations to emulate non-determinism. The oracle technique
+cannot observe the difference in its behaviours.                      can be similarly applied to the Haskell executable specifi-
+   For example, the original Cogent signature of the function         cation. When we test that an oracle-carrying mock refines
+ostore_read is:                                                       a non-deterministic specification, the specification can ab-
+  type RR x a e = (x, <Success a | Error e>)
+                                                                      stract over the values that the oracle can possibly possess
+  ostore_read : (SysState, MountState!, OstoreState, ObjId)           with the NonDet monad. If the specification is to be made
+             → RR (SysState, OstoreState) Obj ErrCode                 more precise, we can also introduce an oracle to it to ascribe
+The function takes a quadruple as input, containing a read-           the source of the non-determinism. In this case, care needs
+only (denoted by the ! operator) MountState, and returns the          to be taken to ensure that the oracle in the specification and
+parametric RR type, which is further defined as a pair of a           that in the implementation are in synchronisation, so that
+variant type <Success a | Error e> and a result type x that           they do not make conflicting choices.
+is common to both cases.                                                 Using oracles in the specification can be problematic if
+   The purely functional nature of Cogent makes it easy               the implementation is not a mock and is genuinely non-
+for developers to identify the observable behaviours—they             deterministic due to, say, the hardware or the operating sys-
+are necessarily within the return type of a function. In the          tem. There is in general no way to predict accurately which
+case of the fsop_readpage function, the only behaviours               1 In our implementation, we pass the oracle around using GHC Haskell’s
+of ostore_read that can be observed are the returned Obj              implicit parameter extension [48], making it more transparent to users. In
+value or the error code ErrCode. Therefore, we can tailor             the presentation of this paper, however, we pass them explicitly for clarity.
+
+
+
+
+                                                                 92
+SLE ’22, December 06–07, 2022, Auckland, New Zealand                Z. Chen, C. Rizkallah, L. O’Connor, P. Susarla, G. Klein, G.Heiser, and G. Keller
+
+
+execution path the concrete implementation will take (e.g.               6.5    Results
+malloc failures). Hence, the specification and the implemen-             The shape of the top-level refinement statement for the
+tation can make inconsistent choices when they encounter                 Haskell shallow embedding of the Cogent fsop_readpage
+non-determinism, which can lead to spurious test failures. In            function (shown below) closely resembles that of the
+this case, we have to step back and use a non-deterministic              WordArray example. An oracle is also generated and passed to
+specification with the NonDet monad instead. This, however,              the Haskell embedding, which will be further passed to the
+has a negative cosmetic effect on the entire Haskell specifica-          mock implementation of ostore_read as discussed earlier.
+tion, as the NonDet monad is infectious and it would render
+                                                                             prop_corres_fsop_readpage :: Property
+all ancestor functions in the call graph monadic.
+                                                                             prop_corres_fsop_readpage =
+   To address this problem, we can establish an equivalence                    forAll gen_fsop_readpage_arg $ \ic →
+relation between the non-deterministic specification using                       forAll gen_oracle $ \o →
+NonDet and the oracle-carrying deterministic one by testing.                       let ia = abs_fsop_readpage_arg ic
+                                                                                       oa = uncurry4 hs_fsop_readpage ia
+Concretely, we test that the two specifications return the
+                                                                                       oc = fsop_readpage o ic
+same set of results, by enumerating every possible oracle                           in corres rel_fsop_readpage_ret oa oc
+in the deterministic specification, collecting a finite set of
+results, and then checking that set against the set of results           From the counter-examples produced by QuickCheck, we
+produced by the non-deterministic specification.                         found that the Haskell executable specification was flawed,
+   The NonDet monad and the oracle approach are two ex-                  which in turn exposed a problem with the Isabelle/HOL
+tremes of the spectrum, and developers can choose a suitable             abstract specification, from which the Haskell specification
+degree of non-determinism by combining these two to meet                 was derived. These specifications did not take errors returned
+the needs of a specific test case.                                       from ostore_read into account. Testing the above property
+                                                                         helped us rectify this mis-specification.
+
+                                                                         7     Design Decisions and Key Takeaways
+                                                                         We have showcased two applications of the PBT framework
+6.4   Test Data Generation                                               in Cogent. Since the examples we examined are from a real
+By using mocks, not only can we use simpler algorithms to                file system, they have given us some good insights into how
+simulate the functionality of the original components, but we            much boilerplate code is required, which components of the
+can also fine-tune test data generators to restrict the domain           testing infrastructure can be automatically generated, and
+of inputs given to the mock, allowing us to only implement               how these pieces can be integrated.
+a partial mock of the original code.
+   When testing a cluster of functions and the mock func-                7.1    Modular Testing and Whole-System Testing
+tion’s input depends on the output of other functions, the               Our QuickCheck machinery does not require the user to
+aforementioned partial mock should be used with care. The                test the entire system at once. Instead, the user may test
+input to a function can be directly controlled by the tester             refinement for each function or for a cluster of functions at
+by defining appropriate test data generators, while the out-             a time. Typically, the ADTs implemented in C form a com-
+put of a function is not so easy to predict as it may have               mon module, shared across many systems. Accordingly, our
+been heavily processed and manipulated by the functions.                 framework allows developers to test the ADTs in isolation,
+When such an input value reaches the partial mock imple-                 with no regard to how they are used within systems. This
+mentation, it is harder to know a priori whether it falls into           modularity is aided by Cogent’s functional semantics.
+the unhandled sub-domain of the mock. It poses a greater                    For the fsop_readpage example, we chose to employ modu-
+challenge in writing good test data generators to ensure the             lar testing as opposed to whole-system testing, which would
+pre-condition of the mock function is met even after the ran-            have required extra effort in developing the infrastructure
+domly generated data have flowed through other functions                 to run tests in kernel mode (see Section 7.3). Whole-system
+in the system under test. More general remarks on this point             testing allows for more abstract top-level properties to be
+can be found in Section 7.4.                                             specified: for instance, that read and write are inverse opera-
+   Domain-specific knowledge can be leveraged to write                   tions.2 Alternatively, these logical properties can be tested
+good test data generators, which makes the checking pro-
+cess more efficient and practicable. For example, when we                2 It might be surprising to some readers that we claim that this property
+
+generate the OstoreState, all entries we generate belong                 is suitable for whole-system testing, instead of PBT. After all, this kind
+to the same inode. In reality, there are many data objects               of round-trip properties are typical in the realm of PBT. The reason is
+                                                                         that, systems software, or file systems in this case, are not implemented
+for other files, or other types of objects; but none of these            cleanly in a purely functional manner. They always involve heavy I/O,
+facts will be observed by its caller. This in turn simplifies the        kernel interaction, locking, etc., which cannot be precisely and concisely
+implementation of the mock and the abstraction functions.                specified in the functional specification and thus fall under the global state.
+
+
+
+
+                                                                    93
+Property-Based Testing: Climbing the Stairway to Verification                            SLE ’22, December 06–07, 2022, Auckland, New Zealand
+
+
+on top of the executable specification rather than directly on         such as KML [51], House [34] or HaLVM [35] to run PBT in
+the concrete implementation.                                           kernel mode. This would allow the testing environment to
+   As our specification becomes more abstract, the single              more closely resemble the real run-time environment of the
+step simulation style of refinement becomes less relevant,             software. We leave it for future investigation.
+because several low-level functions may be specified as a
+single function on the abstract level. Modular testing, on             7.4    Test Generation Strategies
+the other hand, is more comprehensive, as it also examines             There are two main factors to consider when generating test
+the interfaces among different components in a system. In              data for PBT. The first is how to sample the data: we choose
+this case, it uncovered issues in the WordArray implementa-            user-guided random test generation à la QuickCheck in this
+tion that whole-system testing would have, and indeed had,             work. Exhaustive testing (for small values) is another popular
+missed.                                                                strategy and has gained great popularity, e.g. SmallCheck for
+                                                                       Haskell [60]. However, the small scope hypothesis on which
+7.2   Functional Specification Versus Logical                          SmallCheck is based does not hold in general in the context
+      Properties                                                       of systems software.3 For instance, integer overflow, which
+Traditional PBT tests specifications against a set of logi-            is a common bug in systems code, can hardly be triggered
+cal properties (e.g. get and set are inverse operations on             by small values. The second main concern is the effective
+WordArrays). We instead test functions against a full exe-             generation of test data which satisfies the premises of the
+cutable specification that models the functions. This is con-          properties. If a property has the form 𝑝 =⇒ 𝑞 and the premise
+ceptually similar to model-based testing [43] (also see Sec-           𝑝 is very strong (i.e. difficult to satisfy), and the test data is
+tion 8). The functional specification is most akin to the func-        not sampled with great care, then a lot of them will falsify
+tional notions of model paradigm as classified in the work             the premise and thus be discarded in the test, rendering the
+by Utting et al. [64, § 3.3].                                          test inefficient. All refinement statements in this work have
+   It is often easier to use a model rather than a set of prop-        the form 𝑝 =⇒ 𝑞 with a strong precondition 𝑝. The Luck
+erties to define the behaviour of functions [43]; our exper-           framework [46] couples the predicates of the property and
+iments concurred. For example, functions in the C library              the test data generation, which could simplify writing custom
+can be readily modeled in terms of Haskell library functions.          test data generators. There is a rich body of research devoted
+Moreover, low-level functions in systems programming, e.g.             to test generation techniques [28]. In the future, we plan to
+setting a flag, are often very simple in its functionality, but        explore more options to automate our test data generators.
+can hardly be characterised by traditional properties that are
+abstract and intuitive enough for users to comprehend.                 7.5    Shrinking
+   Using functional specifications encourages composition-             Counter-example shrinking reduces the size of counter-
+ality. The functional specification of one module can also             examples before reporting them to testers, which helps
+be used as a mock implementation when testing other                    developers better understand and fix bugs. The Haskell
+modules that depend on this module. For instance, in our               QuickCheck provides a customisable shrinking library with
+fsop_readpage case study, we used the previously defined               a default shrinking algorithm for many datatypes. A rich
+Haskell functional specification of the WordArray functions            body of research can be found on more advanced shrinking
+as mocks.                                                              algorithms. For example, test data shrinking that preserve
+   Furthermore, functional specifications serve as a commu-            invariants about the generated data has been explored in
+nication interface between system programmers and proof                [49, 62]. But due to the lack of recursively defined datatypes
+engineers. They are key to designing verification-friendly             in Cogent and thus in the Cogent-powered file systems, the
+systems programs, whereas logical properties alone fall short          effectiveness of shrinking is dubious, as the size of the input
+in this aspect.                                                        data chiefly comes from the sheer complexity of the (non-
+                                                                       recursive) datatypes, rather than from recursion. Shrinking
+7.3   Testing Kernel Modules
+                                                                       is nevertheless useful for testing ADTs, but basic shrinking
+A file system is typically compiled as a kernel module and             strategies work reasonably well in our context.
+runs in kernel mode, while our test framework runs in user
+mode. To handle this discrepancy, for our prototype, we have           8     Related Work
+ported our file systems code to run in user mode, using mocks
+                                                                       QuickCheck has been used for testing a variety of high-level
+to simulate the kernel APIs. Emulating the kernel is common
+                                                                       properties, such as information flow control [23, 37], mutual
+practice in systems programming, with libraries such as
+FUSE [31] facilitating user-space execution of kernel code.            3 The small scope hypothesis is stated in Runciman et al. [60]’s paper as: “(1)
+However, these tools expect a complete kernel module, thus             If a program fails to meet its specification in some cases, it almost always
+precluding the use of mocks or other user-land code during             fails in some simple case. Or in contrapositive form: (2) If a program does
+testing. A possible alternative to explore is using a system           not fail in any simple case, it hardly ever fails in any case.”
+
+
+
+
+                                                                  94
+SLE ’22, December 06–07, 2022, Auckland, New Zealand                Z. Chen, C. Rizkallah, L. O’Connor, P. Susarla, G. Klein, G.Heiser, and G. Keller
+
+
+exclusion [21], and the functional correctness of AUTOSAR                   In the PBT framework we presented, as we test the re-
+components [6, 53]. To the best of our knowledge, our frame-             finement statement between the implementation and the
+work is the first to use PBT for testing refinement-based                Haskell executable specification, which can be considered
+functional correctness statements.                                       a conformance relation, it does appear that we are instead
+   The hs-to-coq tool [61] translates Haskell code into the              conducting model-based testing [63, 64]. Our approach does
+Coq proof assistant [13]. Breitner et al. [14] used it to verify         indeed share a lot in common with MBT, but we identify
+parts of Haskell’s container library in Coq. In addition to              our approach as PBT for the following reasons. Firstly, in
+proving the functional correctness of various functions in               MBT, the starting point of testing is a model of the software
+the library, they also verified that the QuickCheck proper-              under test. In contrast, as we have demonstrated, testing in
+ties that the library is tested against are correct. By contrast,        our framework does not necessarily have an existing model
+our QuickCheck properties are refinement properties that                 to start with. In developing formally verifiable operating
+directly resemble the those used for full verification. Veri-            systems components, which is the application domain that
+fying these properties is already a substantial step towards             concerns us, it is of paramount importance to find the right
+proving functional correctness, and in some cases directly               balance between verifiability and performance. PBT gives
+implies functional correctness.                                          developers insights in both aspects. Therefore, testing plays
+   QuickCheck is available as a built-in tool in Isabelle/HOL            a role in the design of the system, and subsequently its spec-
+and is used for quickly finding counter-examples to proposed             ification. This is similar to the iterative development process
+lemmas [10, 15]. We chose to build on Haskell’s QuickCheck               reported in the seL4 formal verification work [36]. Secondly,
+rather than Isabelle/HOL’s QuickCheck because it is easier               test cases are systematically and algorithmically generated
+for Cogent programmers to use a testing framework that                   from the model in MBT. Test inputs are typically concretised
+lies in the ecosystem of a functional programming language               from the abstract test suite and the test results are abstracted
+rather than interact with a theorem prover. Haskell acts as              to be validated against the model by an adapter. In contrast,
+a good communication medium between programmers and                      as we have shown in the examples, our test cases are not gen-
+proof engineers [14, 26]. Moreover, due to Isabelle/HOL’s                erated from the specification; test data is directly produced
+interactive nature, testers would have to wait for Isabelle to           on the concrete level. Lastly, from the tooling perspective,
+re-process the proof scripts affected by a change in a theory            our approach uses a PBT library QuickCheck as the core of
+file, before they can run tests again. Even if Isabelle’s quick-         the testing infrastructure.
+and-dirty mode is enabled, which skips proofs, testers would
+still have to wait for Isabelle/HOL to process definitions. In
+fact, a large portion of the time is spent on reading in the             9     Conclusion
+deep embeddings of the Cogent program into Isabelle, due                 In this paper, we showed how we augmented the Cogent
+to the large terms generated by the Cogent compiler. This                verification framework with PBT. Testing and formal verifi-
+would cause a significant and unnecessary reduction to their             cation complement each other, which is well acknowledged
+productivity, and destroy the user experience.                           among researchers and developers. In this work, we further
+   The SPARK language [2], a formally defined subset of Ada,             demonstrated this common belief in the specific context of
+also uses a combination of testing and verification to facil-            PBT and interactive theorem proving. The central idea is to
+itate the development of high-reliability software. SPARK                mirror the refinement proof in testing, using a functional
+developers can attach contracts, that is, specifications of pre-         specification as the model instead of a set of logical proper-
+and postconditions, to critical procedures. Tools of the frame-          ties as commonly done in PBT.
+work can use these contracts as input to automatically test                 Using this method, we tested an abstract data type from a
+the procedures, or attempt to formally prove that the imple-             library, as well as an operation of a real-world file system.
+mentation observes these contracts. Ada language features                The tests exposed several bugs in the ADT implementation
+that are hard to verify, such as side-effects in expressions,            and uncovered errors in the specification of the ADT and of
+access types, allocators, exception handling and many others,            the file system.
+are not permitted in SPARK. SPARK focuses on selectively                    Besides the main purpose of testing—detecting bugs—
+verifying safety critical components, rather than fully veri-            we exhibited other benefits of employing PBT. It reduces
+fied systems from high-level specification to machine code.              the effort in formal verification, guides the development of
+   DoubleCheck [30] integrates PBT into Dracula [65], a ped-             verification-ready specifications and programs, and acts as
+agogical programming environment which enables students                  a precise and effective communication media among devel-
+to develop programs and then prove theorems about them in                opers. We believe PBT offers developers the opportunity
+ACL2 [1], a theorem prover based on term rewriting. As with              to gradually tackle the verification challenge in large and
+our work, the motivation of this integration is to facilitate            complex systems development, serving as a helpful stepping
+formal verification, though its focus is on education, not on            stone in the endeavour into full formal verification of high
+producing verified real-world applications.                              assurance software.
+
+
+
+                                                                    95
+Property-Based Testing: Climbing the Stairway to Verification                                          SLE ’22, December 06–07, 2022, Auckland, New Zealand
+
+
+References                                                                           [19] Louis Cheung, Liam O’Connor, and Christine Rizkallah. 2022. Over-
+ [1] ACL2. 2022. ACL2. Retrieved October 2022 from http://www.cs.                         coming Restraint: Composing Verification of Foreign Functions with
+     utexas.edu/users/moore/acl2/                                                         Cogent. In Proceedings of the 11th ACM SIGPLAN International Confer-
+ [2] AdaCore. 2022. SPARK Pro. Retrieved October 2022 from https:                         ence on Certified Programs and Proofs (CPP 2022). ACM, New York, NY,
+     //www.adacore.com/sparkpro/                                                          USA, 13–26. https://doi.org/10.1145/3497775.3503686
+ [3] Sidney Amani. 2016. A Methodology for Trustworthy File Systems. PhD             [20] Koen Claessen and John Hughes. 2000. QuickCheck: A Lightweight
+     Thesis. CSE, UNSW, Sydney, Australia.                                                Tool for Random Testing of Haskell Programs. In Proceedings of the
+ [4] Sidney Amani, Alex Hixon, Zilin Chen, Christine Rizkallah, Peter                     5th International Conference on Functional Programming. 268–279.
+     Chubb, Liam O’Connor, Joel Beeren, Yutaka Nagashima, Japheth Lim,               [21] Koen Claessen, Michal Palka, Nicholas Smallbone, John Hughes, Hans
+     Thomas Sewell, Joseph Tuong, Gabriele Keller, Toby Murray, Gerwin                    Svensson, Thomas Arts, and Ulf Wiger. 2009. Finding Race Conditions
+     Klein, and Gernot Heiser. 2016. Cogent: Verifying High-Assurance File                in Erlang with QuickCheck and PULSE. In International Conference on
+     System Implementations. In International Conference on Architectural                 Functional Programming. ACM, New York, NY, USA, 149–160. http:
+     Support for Programming Languages and Operating Systems. Atlanta,                    //doi.acm.org/10.1145/1596550.1596574
+     GA, USA, 175–188.                                                               [22] David Cock, Gerwin Klein, and Thomas Sewell. 2008. Secure Microker-
+ [5] Sidney Amani and Toby Murray. 2015. Specifying a Realistic File                      nels, State Monads and Scalable Refinement. In Proceedings of the 21st
+     System. In Workshop on Models for Formal Analysis of Real Systems.                   International Conference on Theorem Proving in Higher Order Logics.
+     Suva, Fiji, 1–9.                                                                     Springer, Montreal, Canada, 167–182.
+ [6] Thomas Arts, John Hughes, Ulf Norell, and Hans Svensson. 2015. Test-            [23] Arthur Azevedo de Amorim, Nathan Collins, André DeHon, Delphine
+     ing AUTOSAR software with QuickCheck. In International Conference                    Demange, Cătălin Hriţcu, David Pichardie, Benjamin C. Pierce, Randy
+     on Software Testing, Verification and Validation (ICST) Workshops. Graz,             Pollack, and Andrew Tolmach. 2014. A Verified Information-Flow
+     AT, 1–4. https://doi.org/10.1109/ICSTW.2015.7107466                                  Architecture. In ACM SIGPLAN-SIGACT Symposium on Principles of
+ [7] R. J. R. Back. 1988. A calculus of refinements for program derivations.              Programming Languages. San Diego, CA, USA, 165–178.
+     Acta Informatica 25, 6 (Aug. 1988), 593–624. https://doi.org/10.1007/           [24] Willem-Paul de Roever and Kai Engelhardt. 1998. Data Refinement:
+     BF00291051                                                                           Model-Oriented Proof Methods and their Comparison. Number 47 in
+ [8] Erik Barendsen and Sjaak Smetsers. 1993. Conventional and Unique-                    Cambridge Tracts in Theoretical Computer Science. Cambridge Uni-
+     ness Typing in Graph Rewrite Systems. In Foundations of Software                     versity Press, United Kingdom.
+     Technology and Theoretical Computer Science (Lecture Notes in Com-              [25] Edsko de Vries, Rinus Plasmeijer, and David M. Abrahamson. 2008.
+     puter Science, Vol. 761). 41–51.                                                     Uniqueness Typing Simplified. In Implementation and Application of
+ [9] Brian Behlendorf. 2011. POSIX Filesystem Test Suite. Retrieved August                Functional Languages (Lecture Notes in Computer Science, Vol. 5083).
+     2022 from https://github.com/zfsonlinux/fstest                                       Springer, 201–218.
+[10] Stefan Berghofer and Tobias Nipkow. 2004. Random Testing in Is-                 [26] Philip Derrin, Kevin Elphinstone, Gerwin Klein, David Cock, and
+     abelle/HOL. In Proceedings of the Software Engineering and Formal                    Manuel M. T. Chakravarty. 2006. Running the Manual: An Approach
+     Methods, Second International Conference (SEFM ’04). IEEE Computer                   to High-Assurance Microkernel Development. In Proceedings of the
+     Society, Washington, DC, USA, 230–239. http://dx.doi.org/10.1109/                    ACM SIGPLAN Haskell Workshop. Portland, OR, USA.
+     SEFM.2004.36                                                                    [27] Oscar Downing. 2021. Enhancements to the Cogent Property-Based
+[11] Jean-Philippe Bernardy, Mathieu Boespflug, Ryan R. Newton, Simon                     Testing Framework. Undergraduate Thesis. CSE, UNSW, Sydney, Aus-
+     Peyton Jones, and Arnaud Spiwack. 2017. Linear Haskell: Practical                    tralia. https://people.eng.unimelb.edu.au/rizkallahc/theses/oscar-
+     Linearity in a Higher-order Polymorphic Language. Proc. ACM Pro-                     downing-honours-thesis.pdf
+     gram. Lang. 2, POPL (Dec. 2017), 5:1–5:29. http://doi.acm.org/10.1145/          [28] Jonas Duregård, Patrik Jansson, and Meng Wang. 2012. Feat: Functional
+     3158093                                                                              Enumeration of Algebraic Types. In Proceedings of the 2012 Haskell
+[12] Jean-Philippe Bernardy, Patrik Jansson, and Koen Claessen. 2010. Test-               Symposium (Haskell ’12). ACM, New York, NY, USA, 61–72. http:
+     ing Polymorphic Properties. In Programming Languages and Systems.                    //doi.acm.org/10.1145/2364506.2364515
+     Springer Berlin Heidelberg, Berlin, Heidelberg, 125–144.                        [29] Peter Dybjer, Haiyan Qiao, and Makoto Takeyama. 2003. Combining
+[13] Yves Bertot and Pierre Castéran. 2004. Interactive Theorem Proving and               Testing and Proving in Dependent Type Theory. In Theorem Proving
+     Program Development. Coq’Art: The Calculus of Inductive Constructions.               in Higher Order Logics. Springer Berlin Heidelberg, Berlin, Heidelberg,
+     Springer.                                                                            188–203.
+[14] Joachim Breitner, Antal Spector-Zabusky, Yao Li, Christine Rizkallah,           [30] Carl Eastlund. 2009. DoubleCheck Your Theorems. In Proceedings of
+     John Wiegley, and Stephanie Weirich. 2018. Ready, set, verify! applying              the Eighth International Workshop on the ACL2 Theorem Prover and
+     hs-to-coq to real-world Haskell code (experience report). PACMPL 2,                  Its Applications (ACL2 ’09). ACM, New York, NY, USA, 42–46. http:
+     ICFP (2018), 89:1–89:16.                                                             //doi.acm.org/10.1145/1637837.1637844
+[15] Lukas Bulwahn. 2012. The New Quickcheck for Isabelle: Random,                   [31] FUSE. 2022. The FUSE Project. Retrieved October 2022 from https:
+     Exhaustive and Symbolic Testing Under One Roof. In International                     //github.com/libfuse/libfuse
+     Conference on Certified Programs and Proofs. Springer-Verlag, Berlin,           [32] GHC. 2022. GHC User’s Guide. Retrieved October 2022 from https:
+     Heidelberg, 92–108. http://dx.doi.org/10.1007/978-3-642-35308-6_10                   //downloads.haskell.org/ghc/latest/docs/users_guide/
+[16] Manuel M. T. Chakravarty. 1999. C → Haskell, or Yet Another Interfac-           [33] Havva Gulay Gurbuz and Bedir Tekinerdogan. 2018. Model-based
+     ing Tool. In Implementation of Functional Languages, 11th International              testing for software safety: a systematic mapping study. Software
+     Workshop, IFL’99, Lochem, The Netherlands, September 7-10, 1999, Se-                 Quality Journal 26, 4 (Dec 2018), 1327–1372. https://doi.org/10.1007/
+     lected Papers. 131–148. https://doi.org/10.1007/10722298_8                           s11219-017-9386-2
+[17] Zilin Chen. 2022. Cogent property-based testing case studies. https:            [34] Thomas Hallgren, Mark P. Jones, Rebekah Leslie, and Andrew Tol-
+     //github.com/au-ts/cogent/tree/master/impl/fs/bilby/quickcheck.                      mach. 2005. A principled approach to operating system construction
+[18] Zilin Chen, Liam O’Connor, Gabriele Keller, Gerwin Klein, and Gernot                 in Haskell. In Proceedings of the 10th International Conference on Func-
+     Heiser. 2017. The Cogent Case for Property-Based Testing. In Work-                   tional Programming. Tallinn, Estonia, 116–128.
+     shop on Programming Languages and Operating Systems (PLOS). ACM,                [35] HaLVM. 2018. The Haskell Lightweight Virtual Machine (HaLVM)
+     Shanghai, China, 1–7.                                                                source archive. Retrieved October 2022 from https://github.com/
+                                                                                          GaloisInc/HaLVM
+
+
+
+                                                                                96
+SLE ’22, December 06–07, 2022, Auckland, New Zealand                             Z. Chen, C. Rizkallah, L. O’Connor, P. Susarla, G. Klein, G.Heiser, and G. Keller
+
+
+[36] Gernot Heiser, June Andronick, Kevin Elphinstone, Gerwin Klein, Ihor                  every-language
+     Kuz, and Leonid Ryzhyk. 2010. The Road to Trustworthy Systems.                   [51] Toshiyuki Maeda. 2015. Kernel Mode Linux: Execute user processes
+     In ACM Workshop on Scalable Trusted Computing (ACMSTC). ACM,                          in kernel mode. Retrieved October 2022 from http://web.yl.is.s.u-
+     Chicago, IL, USA, 3–10.                                                               tokyo.ac.jp/~tosh/kml/
+[37] Cătălin Hriţcu, John Hughes, Benjamin C. Pierce, Antal Spector-                  [52] Carroll Morgan. 1990. Programming from Specifications (2nd ed.).
+     Zabusky, Dimitrios Vytiniotis, Arthur Azevedo de Amorim, and                          Prentice Hall.
+     Leonidas Lampropoulos. 2013. Testing Noninterference, Quickly. In                [53] Wojciech Mostowski, Thomas Arts, and John Hughes. 2017. Modelling
+     International Conference on Functional Programming. 455–468.                          of Autosar Libraries for Large Scale Testing. In Workshop on Models
+[38] John Hughes. 2016. Experiences with QuickCheck: Testing the Hard                      for Formal Analysis of Real Systems (MARS@ETAPS). 184–199. https:
+     Stuff and Staying Sane. In A List of Successes That Can Change the World.             //doi.org/10.4204/EPTCS.244.7
+     Lecture Notes in Computer Science, Vol. 9600. Springer, 169–186.                 [54] Tobias Nipkow and Gerwin Klein. 2014. Concrete Semantics with
+[39] Steve Klabnik and Carol Nichols. 2017. The Rust Programming Lan-                      Isabelle/HOL. Springer.
+     guage. No Starch Press.                                                          [55] Liam O’Connor. 2019. Type Systems for Systems Types. Ph. D. Disser-
+[40] Gerwin Klein, June Andronick, Kevin Elphinstone, Toby Murray,                         tation. UNSW, Sydney, Australia. http://handle.unsw.edu.au/1959.4/
+     Thomas Sewell, Rafal Kolanski, and Gernot Heiser. 2014. Compre-                       64238
+     hensive Formal Verification of an OS Microkernel. ACM Transactions               [56] Liam O’Connor, Zilin Chen, Christine Rizkallah, Sidney Amani,
+     on Computer Systems 32, 1 (Feb. 2014), 2:1–2:70.                                      Japheth Lim, Toby Murray, Yutaka Nagashima, Thomas Sewell, and
+[41] Gerwin Klein, Kevin Elphinstone, Gernot Heiser, June Andronick,                       Gerwin Klein. 2016. Refinement Through Restraint: Bringing Down
+     David Cock, Philip Derrin, Dhammika Elkaduwe, Kai Engelhardt, Rafal                   the Cost of Verification. In International Conference on Functional Pro-
+     Kolanski, Michael Norrish, Thomas Sewell, Harvey Tuch, and Simon                      gramming. Nara, Japan.
+     Winwood. 2009. seL4: Formal Verification of an OS Kernel. In ACM                 [57] Bryan O’Sullivan. 2022. aeson: Fast JSON parsing and encoding. Re-
+     Symposium on Operating Systems Principles. ACM, Big Sky, MT, USA,                     trieved August 2022 from https://hackage.haskell.org/package/aeson
+     207–220.                                                                         [58] Liam O’Connor, Zilin Chen, Christine Rizkallah, Vincent Jackson, Sid-
+[42] Edward A. Kmett. 2022. lens: Lenses, Folds and Traversals. Retrieved                  ney Amani, Gerwin Klein, Toby Murray, Thomas Sewell, and Gabriele
+     August 2022 from https://hackage.haskell.org/package/lens                             Keller. 2021. Cogent: uniqueness types and certifying compilation.
+[43] Pieter Koopman, Peter Achten, and Rinus Plasmeijer. 2012. Model                       Journal of Functional Programming 31 (2021).
+     Based Testing with Logical Properties versus State Machines. In Im-              [59] Christine Rizkallah, Japheth Lim, Yutaka Nagashima, Thomas Sewell,
+     plementation and Application of Functional Languages. Springer Berlin                 Zilin Chen, Liam O’Connor, Toby Murray, Gabriele Keller, and Gerwin
+     Heidelberg, Berlin, Heidelberg, 116–133.                                              Klein. 2016. A Framework for the Automatic Formal Verification of
+[44] Peter Lammich. 2013. Automatic Data Refinement. In Proceedings of                     Refinement from Cogent to C. In International Conference on Interactive
+     the 4th International Conference on Interactive Theorem Proving. Lecture              Theorem Proving. Nancy, France.
+     Notes in Computer Science, Vol. 7998. Springer, 84–99.                           [60] Colin Runciman, Matthew Naylor, and Fredrik Lindblad. 2008. Small-
+[45] Peter Lammich and Andreas Lochbihler. 2018. Automatic Refinement                      check and Lazy Smallcheck: Automatic Exhaustive Testing for Small
+     to Efficient Data Structures: A Comparison of Two Approaches. Journal                 Values. In Proceedings of the First ACM SIGPLAN Symposium on Haskell
+     of Automated Reasoning (Mar 2018). https://doi.org/10.1007/s10817-                    (Haskell ’08). ACM, New York, NY, USA, 37–48. http://doi.acm.org/10.
+     018-9461-9                                                                            1145/1411286.1411292
+[46] Leonidas Lampropoulos, Diane Gallois-Wong, Cătălin Hriţcu, John                  [61] Antal Spector-Zabusky, Joachim Breitner, Christine Rizkallah, and
+     Hughes, Benjamin C. Pierce, and Li-yao Xia. 2017. Beginner’s Luck: A                  Stephanie Weirich. 2018. Total Haskell is reasonable Coq. In Interna-
+     Language for Property-based Generators. In ACM SIGPLAN-SIGACT                         tional Conference on Certified Programs and Proofs. Los Angeles, CA,
+     Symposium on Principles of Programming Languages. ACM, New York,                      USA, 14–27.
+     NY, USA, 114–129. http://doi.acm.org/10.1145/3009837.3009868                     [62] Jacob Stanley. 2022. Hedgehog will eat all your bugs. Open Source
+[47] Leonidas Lampropoulos and Benjamin C. Pierce. 2022. QuickChick:                       Project. Retrieved October 2022 from https://github.com/hedgehogqa/
+     Property-Based Testing in Coq. Retrieved October 2022 from https:                     haskell-hedgehog
+     //softwarefoundations.cis.upenn.edu/qc-current/index.html                        [63] Jan Tretmans. 2011. Model-Based Testing and Some Steps towards Test-
+[48] Jeffrey R. Lewis, John Launchbury, Erik Meijer, and Mark B. Shields.                  Based Modelling. Springer Berlin Heidelberg, Berlin, Heidelberg, 297–
+     2000. Implicit Parameters: Dynamic Scoping with Static Types. In ACM                  326. https://doi.org/10.1007/978-3-642-21455-4_9
+     SIGPLAN-SIGACT Symposium on Principles of Programming Languages.                 [64] Mark Utting, Alexander Pretschner, and Bruno Legeard. 2012. A Tax-
+     ACM, New York, NY, USA, 108–118. http://doi.acm.org/10.1145/325694.                   onomy of Model-Based Testing Approaches. Softw. Test. Verif. Reliab.
+     325708                                                                                22, 5 (aug 2012), 297–312. https://doi.org/10.1002/stvr.456
+[49] David R. MacIver. 2016. Integrated vs Type-based Shrinking. Article. Re-         [65] Dale Vaillancourt, Rex Page, and Matthias Felleisen. 2006. ACL2 in
+     trieved October 2022 from http://hypothesis.works/articles/integrated-                DrScheme. In Proceedings of the Sixth International Workshop on the
+     shrinking                                                                             ACL2 Theorem Prover and Its Applications (ACL2 ’06). ACM, New York,
+[50] David R. MacIver. 2016. QuickCheck in Every Language. Retrieved                       NY, USA, 107–116. http://doi.acm.org/10.1145/1217975.1217999
+     October 2022 from https://hypothesis.works/articles/quickcheck-in-               [66] Philip Wadler. 1990. Linear types can change the world!. In Program-
+                                                                                           ming Concepts and Methods.
+
+
+
+
+                                                                                 97
+

packages/opencode/specs/simulation-research/text/2023-quickercheck.txt

@@ -0,0 +1,716 @@
+                                                                                                                 QuickerCheck
+                                                                          Implementing and Evaluating a Parallel Run-Time for QuickCheck
+                                                                            Robert Krook                                                                       Nicholas Smallbone
+                                                                    krookr@chalmers.se                                                                       nicsma@chalmers.se
+                                                              Chalmers University of Technology                                                        Chalmers University of Technology
+                                                                    Gothenburg, Sweden                                                                       Gothenburg, Sweden
+
+                                                                         Bo Joel Svensson                                                                           Koen Claessen
+                                                                   bo.joel.svensson@gmail.com                                                                 koen@chalmers.se
+                                                                      Lind Art & Technology                                                            Chalmers University of Technology
+arXiv:2404.16062v1 [cs.PL] 17 Apr 2024
+
+
+
+
+                                                                        Stockholm, Sweden                                                                    Gothenburg, Sweden
+
+                                         ABSTRACT                                                                                        1    INTRODUCTION
+                                         This paper introduces a new parallel run-time for QuickCheck, a                                 QuickCheck [5] is a widely known Haskell tool for property-based
+                                         Haskell library and EDSL for specifying and randomly testing prop-                              random testing of programs. First, the programmer writes a prop-
+                                         erties of programs. The new run-time can run multiple tests for a                               erty of the program under test that they expect to always hold.
+                                         single property in parallel, using the available cores. Moreover, if a                          Then, to check the property, QuickCheck generates a number of
+                                         counterexample is found, the run-time can also shrink the test case                             random test cases to exercise the property. If the property always
+                                         in parallel, implementing a parallel search for a locally minimal                               held, the check is reported as successful. If a test case makes the
+                                         counterexample.                                                                                 property fail, a process called shrinking is invoked, which consists
+                                            Our experimental results show a 3–9× speed-up for testing                                    of a greedy search for a (locally) minimal failing test case.
+                                         QuickCheck properties on a variety of heavy-weight benchmark                                       For example, we may be testing an implementation of System F
+                                         problems. We also evaluate two different shrinking strategies; deter-                           [6], where we have the following types and functions:
+                                         ministic shrinking, which guarantees to produce the same minimal                            1   type Expr −− expressions
+                                         test case as standard sequential shrinking, and greedy shrinking,                           2   type Type −− types
+                                         which does not have this guarantee but still produces a locally                             3
+                                         minimal test case, and is faster in practice.                                               4   reduce :: Expr −> Maybe Expr
+                                                                                                                                     5   typeOf :: Expr −> Type
+                                         CCS CONCEPTS                                                                                    The types Expr and Type stand for expressions and types of expres-
+                                         • Computing methodologies → Concurrent algorithms; Shared                                       sions in System F. The function reduce takes one evaluation step, if
+                                         memory algorithms; • Theory of computation → Shared                                             possible. The function typeOf computes the type of an expression.
+                                         memory algorithms; • Software and its engineering → Soft-                                       Subject reduction is a property that says that evaluation of expres-
+                                         ware testing and debugging.                                                                     sions does not cause their type to change. This can be expressed as
+                                                                                                                                         a QuickCheck property as follows:
+                                         KEYWORDS                                                                                    1 prop_Preservation :: Expr −> Property
+                                         property-based testing, quickcheck, testing, parallel functional pro-                       2 prop_Preservation e =
+                                         gramming, haskell                                                                           3   isJust r ==> typeOf e == typeOf (fromJust r )
+                                                                                                                                     4   where
+                                         ACM Reference Format:                                                                       5     r = reduce e
+                                         Robert Krook, Nicholas Smallbone, Bo Joel Svensson, and Koen Claessen.                          Here, the operator ==> specifies a precondition: only tests satisfying
+                                         2023. QuickerCheck: Implementing and Evaluating a Parallel Run-Time for                         isJust r are of interest.
+                                         QuickCheck. In The 35th Symposium on Implementation and Application of                             To run QuickCheck, the user must also supply an Arbitrary
+                                         Functional Languages (IFL 2023), August 29–31, 2023, Braga, Portugal. ACM,
+                                                                                                                                         instance describing how to generate random well-typed Exprs1 (a
+                                         New York, NY, USA, 12 pages. https://doi.org/10.1145/3652561.3652570
+                                                                                                                                         non-trivial task studied in e.g. [7]). QuickCheck will then generate
+                                                                                                                                         a configurable amount of random expressions, which by default is
+                                                                                                                                         100, and evaluate the property for them. In fact, QuickCheck will
+                                         Permission to make digital or hard copies of part or all of this work for personal or           typically evaluate the property even more times, because:
+                                         classroom use is granted without fee provided that copies are not made or distributed
+                                         for profit or commercial advantage and that copies bear this notice and the full citation
+                                                                                                                                              • QuickCheck discards any test case not satisfying the pre-
+                                         on the first page. Copyrights for third-party components of this work must be honored.                 condition isJust r, and continues until it has executed 100
+                                         For all other uses, contact the owner/author(s).                                                       tests satisfying the precondition.
+                                         IFL 2023, August 29–31, 2023, Braga, Portugal
+                                                                                                                                         1 There is no requirement by QuickCheck itself that the generator has to generate
+                                         © 2023 Copyright held by the owner/author(s).
+                                         ACM ISBN 979-8-4007-1631-7/23/08.                                                               well-formed terms. This is primarily required to meaningfully exercise the property in
+                                         https://doi.org/10.1145/3652561.3652570                                                         question.
+IFL 2023, August 29–31, 2023, Braga, Portugal                                                     Robert Krook, Nicholas Smallbone, Bo Joel Svensson, and Koen Claessen
+
+
+     • If a test case fails (for example if the function reduce contains              total number of successful tests3 . So, the distribution of sizes during
+       a bug), shrinking searches for a smaller counterexample by                     testing only depends on the total number of successful tests so far,
+       executing the property on many smaller test cases.                             not on the total number of tests in general. This introduces a small
+                                                                                      but significant data dependency preventing parallel evaluations of
+    All this happens sequentially at the moment in QuickCheck. If
+                                                                                      tests; when a worker runs a test it must know the appropriate size
+the evaluation of the property takes a long time, QuickChecking
+                                                                                      of the test case to run, and the appropriate size depends on whether
+it (and possibly shrinking the counterexample) will take an even
+                                                                                      the previous tests were successful or not. This dependency needs
+longer time. This is time often spent waiting by the programmer,
+                                                                                      to be dealt with somehow in the parallelization.
+perhaps wondering why their computer is roaring like a spaceship
+while only one core is in use. The contribution of this paper is                         Shrinking. When a failing test case is found, QuickCheck
+to propose and practically evaluate a way of performing both the                      searches for a smaller failing test case by applying a process called
+testing phase as well as the shrinking phase of QuickCheck in                         shrinking [10]. The goal of shrinking is to produce a locally minimal
+parallel2 .                                                                           failing test case. Shrinking first produces a list of shrink candidates,
+    Note that our work aims to reduce waiting time for the program-                   variants of the test case that have been reduced in size in a vari-
+mer while checking a single property. There exist frameworks (for                     ety of ways. This list is traversed from left to right until we find
+example tasty [4]) that allow testing of multiple properties and unit                 a new failing test case. Shrinking is then applied recursively on
+tests in parallel or even distributed on a cluster. These are typically               the new failing test case until the current failing test case cannot
+used in regression tests or continuous integration. Our work can                      be reduced anymore. Shrinking does not backtrack in search of a
+not only speed up testing in these settings but also during active                    globally minimal counterexample, but only promises to yield a local
+development, where programmers typically run QuickCheck on a                          minimum.
+single property and wait for the result.                                                 To use shrinking, the user must define a shrink function. For a
+                                                                                      type T, this is a function shrink :: T -> [T] which, given a test
+2    WHAT ARE THE CHALLENGES?                                                         case, produces a list of shrink candidates, i.e. smaller or simpler
+                                                                                      test cases to try. For example, suppose that we are testing System F
+Even though running each test is supposed to be independent,
+                                                                                      again, and the type of expressions is defined as follows:
+and as such testing a property 100 times should be a so-called
+embarrassingly parallel task, in practice parallelizing testing is not            1 data Expr
+so easy. For one, individual tests may interact with each other, but              2   = Var String    −− variable
+luckily in Haskell, we often get the independence guarantees we                   3   | App Expr Expr −− application
+need from pure (or at least thread-safe) code. In this paper, we                  4   | ...           −− other constructors
+assume that the property itself is thread-safe.                                          Then we can define a shrink function as follows:
+   But the biggest problem is that QuickCheck’s algorithm is in-                  1 shrink :: Expr −> [Expr]
+herently sequential. This is not at all obvious at first glance. The              2 shrink (Var _) = []
+problem comes from two features in QuickCheck – adjustment of                     3 shrink (App t u) =
+test size, and shrinking. As we will see, these features introduce a              4   concat [ [ t , u ]
+data dependency: the test case that we should try next depends on                 5           , [ App t ' u | t ' <− shrink t ]
+the result of the previous test. Addressing these dependencies was                6           , [ App t u' | u' <− shrink u ]
+one of the main challenges in parallelizing QuickCheck.                           7           ]
+                                                                                  8 shrink ( ... ) = ... −− other constructors
+   Test size. Many times it is enough to generate a small test input
+to falsify a property. QuickCheck tries to generate smaller inputs                    A Var can not be shrunk further, so we return an empty list of can-
+early on, and gradually increases the size as more and more tests                     didates. A function application App t u, however, can be shrunk
+are passed. This is achieved by QuickCheck supplying the test-data                    further. We can remove the App constructor and one of the subex-
+generator with a size. The generators are free to disregard the size                  pressions, leaving just t or u, which if successful may shrink the
+completely but may use it if they wish to. As an example, the default                 expression considerably. We can also keep the App constructor but
+generator for lists uses the size as an upper bound of the length                     shrink the subexpressions.
+of the generated list. The size of the first test is always 0, while                     Note that QuickCheck always tries the shrink candidates in the
+the default upper bound is 100. If the user specifies that 100 tests                  order they appear in the list, from left to right. Hence it is com-
+should be executed, QuickCheck will make sure that the generator                      mon to return the greedy candidates first, those that remove large
+has been provided with all sizes between 0 and 99. The authors                        parts of the value, as we do in this case. Ordering the shrink list
+point out that so far everything discussed is easily parallelisable.                  appropriately can greatly improve the speed of shrinking.
+   However, properties in QuickCheck can not only succeed or fail,                       We propose to parallelize shrinking in two ways:
+but also discard, which means that a pre-condition in the property                       (1) Greedy shrinking evaluates as many shrinking candidates
+was not fulfilled. A discarded test case is not counted towards the                          in parallel as possible, and as soon as a candidate fails, it
+                                                                                             recursively continues with that candidate. It may be that a
+                                                                                      3 The reason for this is that if the precondition of a property is more likely to succeed
+2 The implementation can be found at https://github.com/Rewbert/quickcheck. The       for small test data sizes, we still want to make sure that we exercise the property on
+authors intend to eventually merge this work into mainline QuickCheck.                larger sizes.
+    QuickerCheck                                                                                                      IFL 2023, August 29–31, 2023, Braga, Portugal
+
+
+            candidate earlier in the shrink list (corresponding to a more         1 prop_metamorphic :: Program −> Property
+            aggressive shrink step) would also have failed if we had              2 prop_metamorphic program = ioProperty $ do
+            waited, and in that case, we may perform a smaller shrink             3    writeFile "p.c" (render program)
+            step than necessary.                                                  4    writeFile "q.c" (render $ mutateInput program)
+        (2) Deterministic shrinking speculatively evaluates test cases in         5   output1 <− compileAndRun "p.c"
+            the search before we know we will need to, but always makes           6   output2 <− compileAndRun "q.c"
+            the same choices as in the sequential case. That is, when a           7   mapM removeFile ["p.c", "q.c" , "p.exe" , "q.exe"]
+            shrink candidate fails, it waits until it knows that no earlier       8   return (mutateOutput output1 == output2)
+            candidate fails                                                              The property executes both the original and modified programs
+    In our evaluation, greedy shrinking is usually faster than determin-              after having first written them to the file system. The file system is
+    istic shrinking.                                                                  cleaned up, after which the outputs are compared. The output of
+                                                                                      the unmodified program is modified to reflect the change described
+    3    QUICKERCHECK                                                                 by the metamorphic relation.
+    We present QuickerCheck via two examples. We point out that the                      Unfortunately, running this property with quickCheckPar will
+    QuickCheck API for writing generators, shrinkers, and properties                  produce extremely strange test failures. The reason is that the prop-
+    remains unchanged, and only the internal evaluation of a property                 erty, while innocent-looking, is not thread-safe. There is an implic-
+    is modified.                                                                      itly shared resource, the file system: if multiple instances of the
+                                                                                      property execute in parallel, they will all write to the same files p.c
+       System F. In Section 1, we saw the property prop_Preservation
+                                                                                      and q.c. This leads to obvious race conditions. There are different
+    :: Expr -> Property for testing subject reduction in System F.
+                                                                                      ways to modify the property such that there are no race conditions,
+    To test this property with sequential QuickCheck we run:
+                                                                                      one of which is to let the property create a temporary directory to
+    > quickCheck prop_Preservation                                                    which intermediary files are written.
+    +++ OK! Passed 100 tests.
+                                                                                  1   −− create a fresh temporary directory based on a baseline name
+       As the property is pure, it is safe to test in parallel using Quick-       2   −− withSystemTempDirectory :: String −> ( FilePath −> IO a) −> IO a
+    erCheck. To do so, we must compile the code with the -threaded                3
+    and -rtsopts flags and pass in the -N option to the run-time sys-             4 prop_metamorphic :: Program −> Property
+    tem, to enable parallelism in GHC. Then all we have to do is invoke           5 prop_metamorphic program = ioProperty $ do
+    quickCheckPar instead of quickCheck.                                          6   withSystemTempDirectory "compiler_output" $ \ dir −> do
+       The output (assuming all tests passed) is                                  7     −− rest of property , now using dir as a
+    > quickCheckPar prop_Preservation                                             8     −− scratch space for temporary files
+    +++ OK! Passed 100 tests.
+                                                                                         If we disregard other implicitly shared resources such as CPU
+      tester 0: 50
+                                                                                      caches, RAM, bandwidth, etc, this property can now be evaluated
+      tester 1: 50
+                                                                                      in parallel by using quickCheckPar.
+       The lines tester 0: 50 and tester 1: 50 show that two                             In general, using QuickerCheck requires three steps. (1) Make
+    threads were used (we happened to limit GHC to using two cores)                   sure that the property is thread-safe (only for properties doing
+    and that they each executed 50 test cases. What is not visible in the             I/O). (2) Compile the program with threading options. (3) Run
+    output is that, since the tests were distributed among two cores,                 quickCheckPar instead of quickCheck.
+    QuickerCheck ran close to twice as fast.
+       Compiler testing. A function that is not necessarily embarrass-                4   QUICKERCHECK DESIGN AND
+    ingly parallel is one that is effectful. To test a compiler it is necessary           IMPLEMENTATION
+    to perform IO actions, such as invoking the compiler under test or                The extensions to QuickCheck described in this paper are designed
+    executing the compiled binary. Testing compilers is non-trivial, but              such that as few observable behaviors as possible are changed. Some
+    a well-studied approach is metamorphic testing [3]. In this approach,             design choices of QuickCheck do not lend themselves nicely to par-
+    assuming a function of type Program -> IO Output that compiles                    allelism, and QuickerCheck tries to make reasonable compromises
+    and runs the program, we define a function mutateProgram ::                       where possible. One notable case of this is the way QuickCheck
+    Program -> Program that mutates the program in some way, and                      computes sizes for a test case. The size is derived from the number
+    then specify how the output should change in response by a func-                  of tests that have passed so far, and the number of tests that have
+    tion mutateOutput :: Output -> Output. Mathematically, the                        been discarded since the last passing test. This means that we can
+    property that should hold is:                                                     not compute the size of a test until we have observed the outcome
+1 −− compileAndRun :: Program −> IO Output                                            of all tests that came before. This sounds sub-optimal for paral-
+2 compileAndRun (mutateProgram p) =                                                   lelization; below, we explain what QuickerCheck does to address
+3   fmap mutateOutput (compileAndRun p)                                               this.
+        In practice, we also need to perform various housekeeping tasks                  Testing. The test loop in ordinary, sequential QuickCheck is a
+    such as writing the program source to a file and cleaning up output               recursive function that maintains a state containing e.g. the count
+    files, so a more realistic property is:                                           of how many tests were executed so far, how many were discarded
+IFL 2023, August 29–31, 2023, Braga, Portugal                                                        Robert Krook, Nicholas Smallbone, Bo Joel Svensson, and Koen Claessen
+
+
+due to a failed pre-condition, etc. It also holds the random seed
+used to generate the test case. It generates and executes one test at
+a time, adjusting the size of the test case whenever a test succeeds,
+but not when a test is discarded. Once a test fails, the test loop
+terminates and a shrinking routine is invoked.
+     The parallel test loop is implemented by running concurrent
+instances of the sequential test loop. The main thread spawns con-
+current testers that evaluate one test after another, and then goes to
+sleep until the testers report that all tests have been executed, too
+many tests were discarded, or a counterexample was found. The
+test loop maintains a state that is updated after every test, recording                     Figure 1: An illustration of how size grows as more and more
+how many tests have been passed so far, the next random seed, and                           tests are passed, and to which worker they are assigned. In
+many other things. In order to facilitate multiple, concurrent testers,                     order to get a fair workload for the concurrent testers a stride
+some of the state has been moved into MVars. As an example, the                             is applied when computing sizes.
+integer representing the number of tests a particular thread has yet
+to run resides in an MVar, enabling other threads to read it if they
+wish to steal work from that thread.                                                            graceful. When a property is aborted as violently as this, by
+     Communication between threads occurs as little as possible in                          raising an asynchronous exception, there is a risk that there will be
+order to not incur synchronization costs. When testing is initiated,                        artifacts left from a test. If a property e.g. creates a new file on the
+the number of tests to run is divided equally between the testers, and                      file system that is normally deleted at the end, an interruption by
+only when one thread has exhausted its budget of tests will it inspect                      an asynchronous exception may make the file erroneously persist.
+the budgets of the concurrent testers. If work-stealing is enabled, a
+thread may then decrement the counter of a sibling tester and run                       1 prop :: Input −> Property
+another test on its own. Each tester has its own random seed that                       2 prop ip = ioProperty $ do
+it splits before running a test, as sharing a seed between all testers                  3   run $ writeFile "temp.txt " (show ip)
+would incur synchronization overheads. Additionally, each tester                        4   −− do some work
+computes the sizes to use for test cases based on their individual                      5   run $ deleteFile "temp.txt " −− we may never execute this
+counters for how many tests they have passed, and how many                                    To address this, we introduce a combinator graceful that takes
+tests they have discarded since the last passing test. In an effort to                      an IO action and a handler. The handler will run if QuickCheck
+explore the same set of sizes as in sequential QuickCheck, they each                        makes the choice to terminate evaluation of the property.
+apply a stride: If we have 𝑘 threads, then thread number 𝑖 uses sizes
+𝑖, 𝑖 + 𝑘, 𝑖 + 2𝑘, 𝑖 + 3𝑘, . . .. This is illustrated in figure 1. A compromise          1   −− graceful :: IO a −> IO ( ) −> PropertyM IO a
+is made when a thread steals a test from a sibling tester, in which                     2
+
+case the local next size is used. This reduces synchronization costs,                   3 prop :: Input −> Property
+as the thread that ran the test doesn’t need to report the result                       4 prop ip = ioProperty $ do
+back to the other thread. With this approach, we explore the same                       5   run $ writeFile "temp.txt " (show ip)
+set of sizes as sequential QuickCheck, except when work stealing                        6   graceful
+happens.                                                                                7     (do −− do some work
+     As an alternative to strides, we have also implemented a strategy                  8         deleteFile "temp.txt " )
+that divides the set of sizes into contiguous segments for each of                      9     ( deleteFile "temp.txt " )
+the testers, by applying an offset to the size computation. There is                           The handler is implemented by intercepting the asynchronous
+a risk, however, that test cases generated by e.g. smaller sizes will                       exception before the worker is restarted and running the handler
+run faster than test cases generated with larger sizes. This would                          before rethrowing the exception. graceful can only capture a spe-
+lead to the concurrent testers finishing their given workloads at                           cific exception thrown internally by QuickCheck. We choose to
+different times. Computing sizes with an offset is implemented and                          implement this dedicated operator like this rather than relying on
+can be chosen by configuring the arguments to quickCheckWith4 ,                             existing bracket functionality, as both user code and QuickCheck
+but the default behavior is to use a stride.                                                might already have code in place to deal with exceptions.
+     When a thread finds a counterexample it wakes up the main                                 graceful can be used not only for shrinking but also for testing.
+thread by writing the used seed and size to an MVar. The main                               When one tester finds a counterexample the concurrent testers will
+thread will then terminate the remaining testers before it shrinks                          be aborted. This combinator will make sure that cleanup occurs
+the counterexample, by delivering asynchronous exceptions. This                             then as well.
+is very abrupt, with the exceptions delivered at the next allocation
+point.                                                                                        Shrinking. The existing shrink loop continually evaluates the
+                                                                                            head of the candidate list until a new counterexample is found, at
+                                                                                            which point the loop recurses, or until the list is empty, at which
+4 The function quickCheckWith is a variant of quickCheck that accepts a configuration       point shrinking is terminated. This is illustrated in figure 2a. The
+parameter where default behavior can be overridden.                                         design of the new loop is very similar.
+QuickerCheck                                                                                             IFL 2023, August 29–31, 2023, Braga, Portugal
+
+
+    Rather than a single thread traversing the candidate list one ele-
+ment at a time, the parallel shrink loop spawns concurrent worker
+threads that cooperate and traverse the same list, now residing in
+an MVar. If any of the concurrent workers finds a new counterex-
+ample, they will update the shared list of candidates and signal to
+their sibling workers that they should stop evaluating their current
+candidate and instead pick a new one from the new list.
+    The behavior of this shrink loop might return a non-deterministic
+result. Whereas the previous loop will always find the first coun-
+terexample in the candidate list, the parallel loop might find a                     (a) Illustration of the existing QuickCheck
+counterexample other than the first one. To emulate the determin-                    shrink-loop. It guarantees to always return
+istic behavior, the new loop can choose to only signal a restart                     the same locally minimal counterexample.
+to those concurrent workers that are evaluating candidates that
+appeared after the current one in the candidate list, and tell them
+to speculatively start shrinking the new counterexample. The other
+workers will keep evaluating their current candidates, and if one of
+them turns out to be a counterexample, the current progress will
+be discarded, and shrinking will continue with the new counterex-
+ample. In this case, we might do some unnecessary work, but we
+will get the same deterministic result. Figure 2b illustrates this and
+how this approach may make us evaluate candidates that we don’t
+need.
+    Another alternative is that when any worker has found a coun-                    (b) The new deterministic shrink-loop
+terexample, all concurrent workers are restarted and told to start                   promises to find the same local minimum
+shrinking the new counterexample, regardless if this was the first                   every time, but it may speculatively eval-
+                                                                                     uate other candidates in its search for the
+counterexample or not. This might lead to a non-deterministic re-
+                                                                                     final counterexample.
+sult, as the path down the rose tree of shrink candidates is not the
+leftmost one, as illustrated in figure 2c. Restarting a worker is done
+by raising an asynchronous exception in the worker. The worker
+will catch this exception and enter the shrink-loop anew, and begin
+to search through the new list of candidates.
+    Repeatedly accessing a shared resource may incur overhead costs.
+If two workers attempt to modify a shared resource at the same
+time, one will have to wait for the other. As the list of candidate
+counterexamples is shared between workers, if candidates are eval-
+uated very fast, it is likely that using more threads will slow down
+shrinking.                                                                           (c) The greedy shrink-loop does not guar-
+                                                                                     antee to find the same local minimum, po-
+5     EVALUATION                                                                     tentially returning a different final coun-
+We evaluate QuickerCheck to answer the following four questions                      terexample.
+
+      • Question 1: Is the sequential performance of the new im-
+                                                                         Figure 2: The three figures above illustrate how the search
+        plementation comparable with QuickCheck?
+                                                                         for a minimized counterexample happened. The dotted line
+      • Question 2: How does the parallel run-time scale as we add
+                                                                         represents the final path to the local minimum, green boxes
+        more cores?
+                                                                         are candidate counterexamples that turned out to not be
+      • Question 3: Can we find bugs faster by using more cores?
+                                                                         new counterexamples, and red boxes are counterexamples
+      • Question 4: Can we shrink counterexamples faster by using
+                                                                         that still falsified the property. Grey boxes are candidate
+        more cores?
+                                                                         counterexamples that were never evaluated.
+      • Question 5: Does the choice of shrinking algorithm affect
+        the quality of shrunk counterexamples?
+  To answer these questions we run properties and collect infor-         intended to represent a diverse set of testing tasks. compiler testing
+mation. We will refer to such properties as benchmarks, and the          and compressid are effectful tasks making use of IO facilities, while
+benchmarks we use are described in the following subsection.             the other tasks are pure.
+                                                                            constant. The benchmark named constant is not one that anyone
+5.1     Benchmarks
+                                                                         would write organically, but its inclusion as a benchmark in this
+We perform all our evaluations using six distinct benchmarks. While      set has a very specific purpose. The underlying property is
+the first benchmark constant is artificial, the other benchmarks are
+    IFL 2023, August 29–31, 2023, Braga, Portugal                                       Robert Krook, Nicholas Smallbone, Bo Joel Svensson, and Koen Claessen
+
+
+                                                                               like this will experience race conditions if multiple threads are used.
+                                                                               We test two alternative implementations that make the property
+                                                                               thread-safe in different ways. The first (tmpfs) generates fresh di-
+                                                                               rectories for each concurrent worker to write such files to, and the
+                                                                               second (nofs) uses pipes to pass values around, never using the file
+                                                                               system.
+
+                                                                                   verse. This property asserts the confluence of the rewrite system
+                                                                               for the Verse Core Calculus [2]. A rewrite system is confluent if,
+                                                                               regardless of which rewrite rules are applied in each step, the result
+                                                                               is always the same, single, normal form.
+                                                                                   The property generates an arbitrary term and applies two arbi-
+                                                                               trary sequences of rewrite rules. If the two resulting normal forms
+    Figure 3: A high-level description of the internal testing loop.           are different, the rewrite system is not confluent and the property
+    The loop begins by generating input and then invoking the                  is falsified.
+    property. After this, the loop inspects the outcome before it
+                                                                                  system f. The system f benchmark is a pure property that gen-
+    either reports having found a counterexample, or loops back
+                                                                               erates arbitrary lambda terms and asserts the subject reduction
+    to repeat all steps. The bottom box and all arrows are part of
+                                                                               property, described in section 1, which states that the type of a
+    the internal testing loop, while the top two boxes are defined
+                                                                               term should not change after performing one reduction of said
+    by the user.
+                                                                               term. The code was taken from Etna, an evaluation platform for
+                                                                               Property-based testing frameworks[8].
+    prop_constant :: ( ) −> Bool
+                                                                                  twee. Twee [9] is a high-performance theorem prover for equa-
+1
+
+    prop_constant ( ) = True
+                                                                               tional logic written in Haskell. A key component is the term index,
+2
+
+
+       The cost in execution time of running a test consists of three          a data structure for finding equations matching a given term. The
+    parts – generating input, running the property, and the machinery          twee benchmark is a pure property stating that, after any sequence
+    of the internal testing loop. This is illustrated in figure 3. As Quick-   of update operations on a term index, the data structure’s invariant
+    erCheck only changes the workings of the testing loop, we want             is preserved.
+    to measure the change in cost of just the testing loop. The above
+    property minimizes the execution time of both the generation of            5.2    Results and Discussion
+    test data and evaluation of the property. Generation and evaluation
+                                                                               Evaluation is done using an Intel I7-10700 8-core CPU with turbo-
+    are constant time as there are no random choices to make dur-
+                                                                               boost turned off. The evaluation system is equipped with 64GB of
+    ing generation and evaluation of the property is trivial. Measured
+                                                                               2933MT/s RAM.
+    changes in the execution speed of QuickCheck vs QuickerCheck on
+                                                                                  We use GHC to compile and execute Haskell code, using
+    this benchmark should primarily be a result of the different testing
+                                                                               the compile-time flags -threaded, -feager-blackholing, and
+    loops.
+                                                                               -rtsopts. We don’t try to mitigate garbage collection costs by in-
+       compiler testing. The underlying property of the compiler test-         creasing the nursery size or try to improve the performance in any
+    ing benchmark asserts that a compiler for an imperative language           other way, as we believe most people use QuickCheck without do-
+    generates correct output. The property is stated as a metamorphic          ing this. All invocations of QuickCheck are made with the chatty
+    relation as described in section 3.                                        flag set to False as printing would otherwise affect the results. In
+       In practice, the property does significantly more work than the         appendix A it is illustrated how chatty affects experimentation.
+    other benchmarks. It generates a type-correct imperative program
+                                                                                  Is the sequential performance of the new implementation compara-
+    and produces several executables that are invoked to assert the
+                                                                               ble with QuickCheck? We answer this by executing each benchmark
+    correctness. The generated programs may include non-terminating
+                                                                               several times both with QuickCheck and QuickerCheck, using only
+    loops, so the property might require some time to execute. Such
+                                                                               one core. We compute the median execution times and compare
+    loops are eventually broken by the property itself after consuming
+                                                                               them. The results are presented in figure 4.
+    too many resources. During evaluation, the property will spend a
+                                                                                  Something that immediately stands out is the huge overhead ex-
+    significant amount of time in external processes.
+                                                                               perienced by the constant benchmark. This benchmark is intended
+        compressid. This benchmark composes the two Unix commands              to act as a worst-case property and illustrate precisely what the
+    gzip and gunzip and verifies that the composition behaves as the           overhead of the new testing loop is. The results indicate that, in
+    identity function. It generates an arbitrary string and invokes gzip,      the worst case, QuickerCheck will incur a penalty of 70%.
+    passes the compressed result to gunzip, and asserts that the final            The other benchmarks all perform some actual workload and
+    output is identical to the input.                                          experience much more modest changes in performance. The system
+        This benchmark comes in three flavors – one is a naive imple-          f property, just like the constant property, is very fast. By running
+    mentation (naive) that writes intermediary values directly to the          many more tests it interacts much more with the new testing loop,
+    file system. Since the file system is a shared resource a property         incurring more of the new costs. This shows up by QuickerCheck
+QuickerCheck                                                                                           IFL 2023, August 29–31, 2023, Braga, Portugal
+
+
+
+
+Figure 4: The performance of sequential QuickerCheck com-
+pared to that of QuickCheck. A number of 1 means that
+                                                                         Figure 5: The acquired speedup relative to the sequential
+there was no difference in performance, whereas a num-
+                                                                         execution time when running tests.
+ber less than 1 indicates that QuickerCheck was faster than
+QuickCheck (e.g. 0.5 shows that QuickerCheck finished in
+half the time). A number greater than 1 indicates that Quick-
+erCheck was slower than QuickCheck.
+
+
+
+requiring 11% more execution time to finish the same workload.
+Some of the workloads experienced no change at all or even got
+slightly faster.
+   Not accounting for the constant benchmark, it appears that there
+is no major change in performance by using sequential Quick-
+erCheck instead of QuickCheck.
+
+   How does the parallel run-time scale as we add more cores? Each
+of the benchmarks is executed several times for each core config-
+uration, the median execution time is computed and the speedup
+relative to the sequential running time is computed. The results are
+presented in figure 5.
+   We first observe that many of the benchmarks scale very well
+until the point where we exhaust the number of physical cores. The
+highest speedup was achieved by the compiler testing benchmark
+                                                                         Figure 6: The acquired speedup relative to the sequential
+which got more than eight times faster. The verse property is not
+                                                                         execution time when searching for a planted bug.
+far behind.
+   The twee benchmark initially scales very well but starts to lose
+momentum when we approach the limit of physical resources.
+                                                                         tests in the same time frame. The results seem to indicate that the
+When hyper-threading is active performance slowly but surely de-
+                                                                         more time a property spends inside the body of the property, the
+grades. The twee benchmark is very data-intensive and frequently
+                                                                         greater the potential speedup.
+moves data around. While two hyper-threads appear to the operat-
+ing system as two CPUs, they are actually two logical threads that          Can we find bugs faster by using more cores? To evaluate this we
+share hardware components required to execute machine instruc-           plant a bug in 4 of the 6 benchmarks and let QuickerCheck run
+tions, such as caches and the system bus. One potential explanation      until it finds the bug. This is repeated 300 times after which the
+for this degradation is that the different testers affect the cache in   median execution time is computed. Figure 6 illustrates the speedup
+unfavorable ways.                                                        acquired relative to the sequential execution time.
+   One noticeable difference between e.g. the compiler testing and          We first note that the system f benchmark doesn’t reach as high
+system f benchmark is that the compiler testing property is signifi-     of a speedup as when we are running tests without a bug enabled.
+cantly slower. The property may spend over a second evaluating           While we can’t say with certainty what to attribute this to, we
+a single test while the system f benchmark may run thousands of          have a pretty good guess of what is happening. The shape of the
+IFL 2023, August 29–31, 2023, Braga, Portugal                                      Robert Krook, Nicholas Smallbone, Bo Joel Svensson, and Koen Claessen
+
+
+curve is the same, except that it is pushed down towards lower
+multiples. There are some new costs associated with starting up the
+parallel test loop, and when we run tests without a bug enabled the
+benchmark is allowed to run for a few seconds, running hundreds of
+thousands of tests. The cost of starting up the test loop is amortized
+over all these tests, while when a bug is enabled there are many
+fewer tests. The bug was found after roughly 200 tests, running for
+just a couple of milliseconds.
+   The overall shape of the twee benchmark is the same, but not
+reaching quite as high of a speedup as when just running tests. The
+compiler testing benchmark acquires a 10x increase in performance,
+outperforming all other evaluated benchmarks. We believe this
+speedup is higher than that achieved in figure 5 because many
+concurrently running tests are aborted when a counterexample
+is found. When evaluating speedup for question two, every test
+that we began evaluating was expected to finish, whereas when we
+evaluated question three, we terminated concurrent testers when
+one of them found a counterexample. We will thus do slightly less         Figure 7: The speedups acquired when using two cores to
+work. We observe the inverse behavior in the system f property,           shrink the compiler testing tests, using the deterministic
+where the concurrent testers have time to run many additional tests       algorithm.
+before they are terminated by a tester who found a counterexample.
+
+   Can we shrink counterexamples faster by using more cores? We
+generate 200 random seeds that we know trigger bugs, such that
+we can replay them to deterministically see the same counterexam-
+ples. We replay the seeds and measure how long it takes to shrink
+them, varying the number of cores and the choice of strategy (de-
+terministic or greedy shrinking). We have done this for the three
+benchmarks compiler testing, twee, and verse. Because it is imprac-
+tical to show all the results, we have picked some subsets of data
+that we find representative of the overall results.
+   The compiler testing results are presented in figure 7, 8, and
+9. Figures 7 and 8 illustrate the relationship between sequential
+and parallel execution time, using two cores. The red dots got
+slower when two cores were used, whereas the blue dots achieved
+a speedup. The further from the line a point lies, the more extreme
+the achieved effect is. From the two figures, we can see that the
+greedy algorithm appears to benefit more experiments and that
+the achieved effects are greater. The blue dots in figure 7 appear to
+tangent a line. This line traces the execution time that is twice as      Figure 8: The speedups acquired when using two cores to
+fast as the sequential one and illustrates the upper bound defined        shrink the compiler testing tests, using the greedy algorithm.
+by Amdahl’s law[1]. The results in figure 8 show some experiments
+crossing this boundary, which is explained by the greedy algorithm
+being able to return a completely different counterexample.
+   As more and more cores are added, the results indicate that more          Figure 9 shows that there is a clear trend of counterexamples with
+and more experiments got slower, while the remaining ones that            a good efficiency not benefiting from parallel shrinking. If there
+achieved a speedup achieved a much greater speedup.                       was not that much extra work to be done from the beginning, the
+   To try and answer which counterexamples may benefit from               existence of more cores does not offer any substantial performance
+parallel shrinking, we plot the efficiency of the shrunk counterex-       improvements.
+amples. The efficiency of a single counterexample is defined as              The results observed from twee (figures 10, 11, and 12) tell a
+the fraction of evaluated candidates that successfully shrunk the         different story. The twee property finishes shrinking in a couple
+counterexample, and as such is a number between 0 and 1. It is            of milliseconds, and using more cores quickly makes all observed
+clear that if the efficiency is one, there is nothing to be gained        counterexamples shrink slower. The efficiency appears to make no
+from parallelism as shrinking becomes a sequential search. As an          difference and we believe that the overhead of the parallel search
+example, the total number of evaluated candidates in figures 2a,          overshadows any benefits of using more cores. The advantage of
+2b, and 2c are 5, 7, and 8 respectively. In all 3 cases the number of     having more cores at one’s disposal appears to mainly be beneficial
+successful shrinks was 3, so the efficiencies are 0.6, 0.42, and 0.375.   in cases where execution will require a non-trivial amount of time.
+QuickerCheck                                                                                            IFL 2023, August 29–31, 2023, Braga, Portugal
+
+
+
+
+Figure 9: This figure illustrates that the closer the efficiency is       Figure 11: The greedy algorithm appears to perform roughly
+to one, the higher the probability that the test will get slower          the same as the deterministic one, with the exception of some
+when shrinking. It also appears that the relative speedup is              tests that did indeed shrink faster.
+higher the lower the efficiency.
+
+
+
+
+                                                                          Figure 12: While the efficiency turned out to be an excellent
+Figure 10: The results indicate that most tests finished shrink-          indicator for whether a test got faster or not for the compiler
+ing very fast when only two cores was used.                               testing benchmark, the same can not be said for twee. Using
+                                                                          16 cores and the greedy algorithm, all tests got slower and
+                                                                          there was quite a spread of efficiencies. The overall efficiency
+   The verse benchmark, much like the compiler testing one, achieves
+                                                                          appears to be much lower, but there is still nothing to be
+a noticeable speedup for the majority of candidates. This is illus-
+                                                                          gained by additional cores.
+trated in figures 13 and 14. The efficiency of the shrinker is depicted
+in figure 15, and shows that there is a slight trend towards can-
+didates with a lower efficiency being more likely to benefit from
+                                                                          distribution of sizes of shrunk counterexamples is different for the
+multiple cores. This benchmark shrinks quite rapidly, and as we
+                                                                          two algorithms. We evaluate this on two benchmarks, compiler
+add more cores, more and more candidates become slower, with the
+                                                                          testing and verse. We collect 300 seeds from the compiler testing
+final number at 16 cores showing that roughly half of the candidates
+                                                                          benchmark and 500 from the verse benchmark. These seeds imme-
+experienced a slowdown.
+                                                                          diately falsify the property, allowing us to shrink them and record
+   Does the choice of shrinking algorithm affect the quality of shrunk    the size using both algorithms. We define the size of a counterex-
+counterexamples? The deterministic shrinking algorithm will al-           ample as the number of constructors in it. We point out that both
+ways yield the same locally minimal counterexample, while the             algorithms produce identical results when only one core is used.
+greedy algorithm may return another local minimum, of a poten-               To compare the results from the two algorithms, we model the
+tially different size. We are interested in finding out whether the       measured sizes as negative binomial distributions. Whereas we
+IFL 2023, August 29–31, 2023, Braga, Portugal                                   Robert Krook, Nicholas Smallbone, Bo Joel Svensson, and Koen Claessen
+
+
+
+
+Figure 13: The speedups acquired with two cores using the               Figure 15: The efficiency of the verse shrinker shows that
+deterministic algorithm, for the verse benchmark.                       there is a slight trend of lower efficiency indicating that there
+                                                                        is a speedup to have by using more cores.
+
+
+
+
+Figure 14: The speedups acquired with two cores using the
+greedy algorithm, for the verse benchmark. It can be observed           Figure 16: The measured sizes are rendered as a histogram,
+that the number of candidates that achieved a speedup in-               together with a model that represents the distribution from
+creased, compared to using the deterministic algorithm.                 which they were drawn.
+
+
+only have one baseline model (the deterministic algorithm), we
+have 16 models representing the greedy algorithm (one for each          that the choice of algorithm does not impact the quality of shrunk
+core configuration). The authors note that in the single-core case,     counterexamples at all.
+the two algorithms are identical. Figure 16 illustrates both the
+measured sizes of the deterministic algorithm, as well as the model     6   RELATED WORK
+representing them.                                                      QuickCheck, having proven itself an extremely useful framework
+   We compare the models representing the greedy algorithm to the       for testing software, has been re-implemented in many program-
+baseline model by computing the entropy between them. Figures           ming languages. It appears that most other implementations don’t
+17 and 18 illustrate the baseline model and the greedy model with       support parallel execution of properties. The only other imple-
+the highest relative entropy. In both measured benchmarks the           mentation we could find that supports parallelism is fsCheck, a
+difference is very small. While the verse benchmark shows little        QuickCheck implementation for testing .NET code. The parallel
+to no difference at all, the compiler testing benchmark has a small     run-time is not described in any paper and the documentation is
+but noticeable difference. This difference is not large enough to say   sparse, but the implementation is discussed in a merge request
+whether the distributions are different or not. The results indicate    introducing the work. The discussion indicates that they initially
+QuickerCheck                                                                                            IFL 2023, August 29–31, 2023, Braga, Portugal
+
+
+                                                                           The Haskell package tasty [4] lets the user define test suites with
+                                                                        individual tests in a suite being of different kinds. A test suite can
+                                                                        simultaneously include e.g. QuickCheck tests, SmallCheck tests,
+                                                                        and unit tests. This is possible by tasty using different test drivers
+                                                                        to execute the tests. tasty can execute individual tests in a test suite
+                                                                        in parallel, but it will not introduce parallelism in the underlying
+                                                                        test drivers. If a test suite contains many tests, with all but one test
+                                                                        terminating very quickly, the majority of execution time will be
+                                                                        sequential, waiting for the longest running test to terminate.
+
+
+
+
+                                                                        7   CONCLUSIONS AND FUTURE WORK
+                                                                        Our results show that parallel testing is beneficial. If the property
+                                                                        being tested is slow to run the expected performance increase is
+                                                                        high, whereas a fast property stands to gain less (but not nothing).
+Figure 17: The figure illustrates the distribution of the base-            Thanks to the natural division of effectful and pure code in
+line samples, as well as the greedy distribution with the high-         Haskell, many properties are immediately able to benefit from
+est relative entropy, from the compiler testing benchmark.              the parallel run-time. We found that with slight modifications to
+                                                                        effectful properties, we could run them in a thread-safe manner.
+                                                                           Parallel shrinking is not as universally beneficial, but can still
+                                                                        yield good results. For all benchmarks evaluated, individual coun-
+                                                                        terexamples could go either way, either experiencing a slowdown
+                                                                        or a speedup. We can not conclude that parallel shrinking is always
+                                                                        beneficial. It depends on not only the property but also the specific
+                                                                        test case. As more cores are added, some counterexamples will get
+                                                                        significantly faster, while the likelihood of your counterexample
+                                                                        shrinking slower increases. There seems to be a good compromise
+                                                                        around using multiple cores, but a lower number. The greedy al-
+                                                                        gorithm appears to offer a greater speedup than the deterministic
+                                                                        one, without compromising on the quality of the counterexamples.
+                                                                           While the work presented in this paper represents a considerable
+                                                                        engineering effort, there are still many lines of future work to
+                                                                        pursue. While implementing the work described in this paper, it
+                                                                        became clear that the ad-hoc way of computing sizes in QuickCheck
+                                                                        does not lend itself nicely to parallelism. It imposes a sequential
+                                                                        ordering to test cases and is tricky to distribute over multiple cores.
+Figure 18: The figure illustrates the distribution of the base-         While we have implemented a best-effort attempt to maintain the
+line samples, as well as the greedy distribution with the high-         previous behavior, it is not a perfect imitation. The authors would
+est relative entropy, from the verse benchmark. The distribu-           like to implement and evaluate several different ways of computing
+tions are practically the same.                                         sizes and reach some conclusions about which strategies are most
+                                                                        efficient.
+                                                                           While the greedy algorithm is allowed to search for the fastest
+used an offset to compute sizes for tests but switched to using a       path to a counterexample, there may well be more efficient algo-
+stride after observing an uneven workload between workers.              rithms still. There is still a bias towards finding earlier paths. It
+   The largest framework for property-based testing by number           would be interesting to see how a random walk would perform.
+of users is the Python package Hypothesis. They have explicitly            Currently, the user must explicitly request parallel QuickCheck
+chosen not to provide support for parallel evaluation of properties     by using quickCheckPar instead of quickCheck. This choice was
+as it can not be determined beforehand whether the function being       made because properties involving I/O can not always be safely ex-
+tested is thread-safe or not. In a non-pure language like Python,       ecuted in parallel. It would be possible to instead have QuickCheck
+this might be a concern, but we believe that this concern is not as     automatically execute tests in parallel when it is safe to do so. For
+severe when it comes to Haskell code. Haskell code is usually split     example, pure properties (not using 𝑖𝑜𝑃𝑟𝑜𝑝𝑒𝑟𝑡𝑦) can always be par-
+up into its pure parts and effectful parts, with the pure parts being   allelized. Properties doing I/O could be marked as thread-safe using
+embarrassingly parallel from the get-go. Effectful code can in many     a special combinator.
+cases be refactored to be thread-safe, such that parallel testing may      We are also working together with the QuickCheck maintainers
+yield positive results.                                                 towards merging this line of work into mainline QuickCheck.
+IFL 2023, August 29–31, 2023, Braga, Portugal                                                       Robert Krook, Nicholas Smallbone, Bo Joel Svensson, and Koen Claessen
+
+
+REFERENCES                                                                                  of an effect this has on a sequential workload. The constant and
+ [1] Gene M. Amdahl. 1967. Validity of the single processor approach to achieving           system f properties run extremely fast, and we observe that with
+     large scale computing capabilities. In American Federation of Information Process-     the chatty flag set to True, QuickerCheck is significantly faster.
+     ing Societies: Proceedings of the AFIPS ’67 Spring Joint Computer Conference, April
+     18-20, 1967, Atlantic City, New Jersey, USA (AFIPS Conference Proceedings, Vol. 30).   The constant property finished evaluating in one-fifth of the time
+     AFIPS / ACM / Thomson Book Company, Washington D.C., New York, NY, USA,                that QuickCheck required.
+     483–485. https://doi.org/10.1145/1465482.1465560
+ [2] Lennart Augustsson, Joachim Breitner, Koen Claessen, Ranjit Jhala, Simon
+                                                                                               As we add cores, it appears that chatty might make the bench-
+     Peyton Jones, Olin Shivers, Guy L. Steele Jr., and Tim Sweeney. 2023. The              marks scale slightly worse, but not a lot, as indicated in figure
+     Verse Calculus: A Core Calculus for Deterministic Functional Logic Program-            20.
+     ming. Proc. ACM Program. Lang. 7, ICFP, Article 203 (aug 2023), 31 pages.
+     https://doi.org/10.1145/3607845
+ [3] Tsong Yueh Chen, S. C. Cheung, and Siu-Ming Yiu. 2020. Metamorphic Testing:
+     A New Approach for Generating Next Test Cases. CoRR abs/2002.12543 (2020).
+     arXiv:2002.12543 https://arxiv.org/abs/2002.12543
+ [4] Roman Cheplyaka. 2013. tasty. https://hackage.haskell.org/package/tasty
+ [5] Koen Claessen and John Hughes. 2000. QuickCheck: a lightweight tool for
+     random testing of Haskell programs. In Proceedings of the Fifth ACM SIGPLAN
+     International Conference on Functional Programming (ICFP ’00), Montreal, Canada,
+     September 18-21, 2000, Martin Odersky and Philip Wadler (Eds.). ACM, New York,
+     NY, USA, 268–279. https://doi.org/10.1145/351240.351266
+ [6] Jean-Yves Girard. 1972. Interprétation fonctionnelle et élimination des coupures de
+     l’arithmétique d’ordre supérieur. Ph. D. Dissertation.
+ [7] Michał H Pałka, Koen Claessen, Alejandro Russo, and John Hughes. 2011. Testing
+     an optimising compiler by generating random lambda terms. In Proceedings of
+     the 6th International Workshop on Automation of Software Test. 91–97.
+ [8] Jessica Shi, Alperen Keles, Harrison Goldstein, Benjamin C Pierce, and Leonidas
+     Lampropoulos. 2023. Etna: An Evaluation Platform for Property-Based Testing
+     (Experience Report). Proceedings of the ACM on Programming Languages 7, ICFP
+     (2023), 878–894.
+ [9] Nicholas Smallbone. 2021. Twee: An Equational Theorem Prover. In Automated
+     Deduction - CADE 28 - 28th International Conference on Automated Deduction,
+     Virtual Event, July 12-15, 2021, Proceedings (Lecture Notes in Computer Science,
+     Vol. 12699), André Platzer and Geoff Sutcliffe (Eds.). Springer, New York, NY, USA,
+     602–613. https://doi.org/10.1007/978-3-030-79876-5_35
+[10] Andreas Zeller and Ralf Hildebrandt. 2002. Simplifying and Isolating Failure-
+     Inducing Input. IEEE Trans. Software Eng. 28, 2 (2002), 183–200. https://doi.org/      Figure 19: The sequential performance of QuickerCheck rel-
+     10.1109/32.988498                                                                      ative to QuickCheck, evaluated with the chatty flag turned
+                                                                                            on and off. The (c) suffix indicates that Chatty was set to True.
+A     THE EFFECT OF CHATTY
+The chatty flag in QuickCheck controls whether QuickCheck
+should continuously print what it is doing or not. While printing
+is helpful in assessing the current progress, it can be a bottleneck
+when it comes to performance.
+   QuickCheck prints the current progress before every test. If you
+run just a few tests every second this is of no concern, but if your
+property is a very fast one it has a huge effect on performance.
+Running 10000 tests per second means that you will print 10000
+times per second, which is significantly more than a human eye
+can observe.
+   QuickerCheck takes a different approach to printing. Since the
+new run-time is multi-threaded anyway, QuickerCheck will spawn
+a separate worker thread whose sole purpose is to periodically
+print the progress to the terminal. The duration of the period can
+be configured, and the default is 200 milliseconds.
+   While the property that now runs 10000 tests in one second
+would previously have printed 10000 times, QuickerCheck would                               Figure 20: The speedup relative to sequential execution time,
+only have printed 5 times. In figure 19 it can be observed how much                         when chatty is enabled.
+

packages/opencode/specs/simulation-research/text/2024-pbt-in-practice.txt

@@ -0,0 +1,797 @@
+                                          Property-Based Testing in Practice
+               Harrison Goldstein                               Joseph W. Cutler                            Daniel Dickstein
+            University of Pennsylvania                       University of Pennsylvania                         Jane Street
+              Philadelphia, PA, USA                            Philadelphia, PA, USA                       New York, NY, USA
+              hgo@seas.upenn.edu                               jwc@seas.upenn.edu                       ddickstein@janestreet.com
+
+                                            Benjamin C. Pierce                        Andrew Head
+                                          University of Pennsylvania            University of Pennsylvania
+                                            Philadelphia, PA, USA                 Philadelphia, PA, USA
+                                          bcpierce@seas.upenn.edu                 head@seas.upenn.edu
+
+ABSTRACT                                                                    Python’s Hypothesis framework [37] had an estimated 500K users
+Property-based testing (PBT) is a testing methodology where users           in 2021 according to a JetBrains survey [32]. Still, there is plenty of
+write executable formal specifications of software components and           room for growth. Half a million Hypothesis users represent only 4%
+an automated harness checks these specifications against many               of the total Python user base, whereas the Hypothesis maintainers
+automatically generated inputs. From its roots in the QuickCheck            estimate [15] that the “addressable market” is at least 25%. (For
+library in Haskell, PBT has made significant inroads in mainstream          comparison, the most popular testing framework, pytest, has 50%
+languages and industrial practice at companies such as Amazon,              market share.)
+Volvo, and Stripe. As PBT extends its reach, it is important to un-            To help move PBT toward wider adoption, the research commu-
+derstand how developers are using it in practice, where they see            nity (ourselves included) needs to better understand the practical
+its strengths and weaknesses, and what innovations are needed to            strengths and weaknesses of PBT and the places where further
+make it more effective.                                                     technical advances are required. Existing work in the software engi-
+    We address these questions using data from 30 in-depth inter-           neering literature has studied how other bug-finding tools are used
+views with experienced users of PBT at Jane Street, a financial tech-       in practice (see §6), but PBT offers a unique set of tools and warrants
+nology company making heavy and sophisticated use of PBT. These             its own investigation. Accordingly, we interviewed PBT users at
+interviews provide empirical evidence that PBT’s main strengths             Jane Street, a financial technology firm that makes significant use
+lie in testing complex code and in increasing confidence beyond             of PBT, to learn how they use PBT, where they see its value, and in
+what is available through conventional testing methodologies, and,          what ways they think it might be improved. Concretely, we aimed
+moreover, that most uses fall into a relatively small number of high-       to answer two main questions:
+leverage idioms. Its main weaknesses, on the other hand, lie in the         RQ1: What are the characteristics of a successful and mature PBT
+relative complexity of writing properties and random data genera-                culture at a software company?
+tors and in the difficulty of evaluating their effectiveness. From these    RQ2: Are there opportunities for future work in the PBT space
+observations, we identify a number of potentially high-impact areas              that are motivated by the needs of real developers?
+for future exploration, including performance improvements, dif-
+ferential testing, additional high-leverage testing scenarios, better       The first question aims both to offer guidance for engineers and
+techniques for generating random input data, test-case reduction,           managers considering whether PBT might fit well in their organi-
+and methods for evaluating the effectiveness of tests.                      zations and to provide a basis for evaluating and comparing PBT
+                                                                            technologies. The second question aims to help shape further re-
+1     INTRODUCTION                                                          search to maximize the impact of PBT.
+                                                                               Our findings contribute a wide range of observations about de-
+Property-based testing (PBT) is a powerful tool for evaluating soft-
+                                                                            velopers’ experiences with PBT, adding nuance to the research
+ware correctness. The process of PBT starts with a developer decid-
+                                                                            community’s understanding of PBT’s real-world usage. Through
+ing on a formal specification that they want their code to satisfy
+                                                                            our interviews, we gleaned several new insights about the situa-
+and encoding that specification as an executable property. An au-
+                                                                            tions in which property-based tests are deployed in practice. We
+tomated test harness checks the property against their code using
+                                                                            found that developers use PBT mainly for testing components of
+hundreds or thousands of random inputs, produced by a generator.
+                                                                            complex systems, expecting the tests to provide greater confidence
+If this process discovers a counterexample to the property—an input
+                                                                            than conventional example-based unit tests yet still run quickly as
+value that causes it to fail—the developer is notified.
+                                                                            part of their normal test suite. Interestingly, we also found that de-
+    The research literature is full of accounts of PBT successes, e.g.,
+                                                                            velopers leverage PBT for the secondary benefit of communicating
+in telecommunications software [2], replicated file [31] and key-
+                                                                            specifications: properties serve as a form of persistent documen-
+value [8] stores, automotive software [3], and other complex sys-
+                                                                            tation, demonstrating the semantics of the software to readers—a
+tems [30]. PBT libraries are available in most major programming
+                                                                            benefit less commonly discussed in the literature. Finally, we found
+languages, and some now have significant user communities—e.g.,
+                                                                            that at Jane Street, PBT is primarily used in “high-leverage” sce-
+ICSE 2024, April 2024, Lisbon, Portugal                                     narios, where properties are especially easy to identify and test.
+2024.                                                                       Beyond deepening our understanding of when and why developers
+ICSE 2024, April 2024, Lisbon, Portugal                            Harrison Goldstein, Joseph W. Cutler, Daniel Dickstein, Benjamin C. Pierce, and Andrew Head
+
+
+reach for PBT, we also found that PBT technology can be improved             trees will be discarded, wasting precious generation time. A better
+in several ways to better support developers. In particular, study           strategy is to hand-craft a generator that only produces valid BSTs,
+participants reported struggling to generate distributions of test           but such generators can be nontrivial to write.
+examples that they were convinced effectively exercised the prop-               If the property ever fails during testing, the failing value is pre-
+erty, and sometimes viewed the process of designing random data              sented to the user. This value might be overly complex, with parts
+generators as a distraction. They also lamented the lack of visible          that are irrelevant to the failure of the property, so most PBT frame-
+feedback on the effectiveness of their testing.                              works provide tools for test-case reduction [36], usually called
+   From these findings, we extract a list of opportunities for future        shrinking [29] in the PBT literature.
+research, including understanding the nuances of PBT performance                The BST example above uses OCaml’s QuickCheck library, but
+requirements, exploring better support for differential testing, and         the general approach—a module under test, a concise property, a
+expanding the high-leverage scenarios in which PBT is most effec-            data generator, and a shrinker—is shared by all PBT tools, from
+tive. We also highlight opportunities around improving languages             the original Haskell QuickCheck [11] to Python’s Hypothesis li-
+for random data generators, designing interfaces for test-case reduc-        brary [37] and beyond. The power of PBT comes from the ability
+tion (“shrinking”), evaluating testing success, and using developer          to test a huge number of system inputs with a single specification
+feedback to improve the testing process.                                     and generator, often uncovering edge and corner cases that the
+   We begin with background on PBT (§2), then present our study              developer might not have considered. It thus hits a useful midpoint
+methodology and discuss of potential threats to validity (§3). We            between example-based unit tests and heavier-weight formal meth-
+present the study’s main results (§4) and detail lessons learned in          ods, retaining the precision of traditional formal specifications and
+the form of observations (§5.1) and research opportunities (§5.2)            their ability to characterize a system’s behavior on all possible in-
+before discussing related work (§6) and concluding (§7).                     puts, but offering quick, best-effort validation instead of requiring
+                                                                             developers to write formal proofs.
+2    BACKGROUND                                                                  Of course, PBT is only one among many testing methodologies.
+Properties are executable specifications of programs. For example,           Two primary alternatives are especially relevant to our study.
+suppose a developer is working on a new implementation of binary                 Example-based unit testing [13] (as supported by pytest, JUnit,
+search trees (BSTs)—tree structures where each internal node is              and other frameworks) evaluates a program by testing it on indi-
+labeled with a data value that is greater than any labels in its left        vidual example inputs. For each input, the developer writes down a
+subtree and less than any in its right subtree. They know that all the       short snippet of code that checks that the program’s output is cor-
+operations on BSTs (insert, delete, etc.) must preserve this validity        rect for this input. (We call this style “example-based unit testing”
+condition: given a valid BST, they should always produce a valid BST.        throughout the paper, rather than just “unit testing,” because PBT
+To enforce this condition, they might write the following test for           is also typically used to test individual units within larger systems.)
+the insert function using OCaml’s Core QuickCheck library [19]:                  Fuzz testing, invented by Miller [4, 39] and popularized by tools
+    let test_insert_maintains_bst () =                                       like AFL [56], also aims to find bugs by randomly generating pro-
+      QuickCheck . test                                                      gram inputs. The technical foundations of fuzz testing and PBT
+        ( both int_gen ( filter valid_bst tree_gen ))                        overlap to a large (and increasing [34, 53]!) degree, but existing tools
+        ( fun (x , t ) -> valid_bst ( insert t x ))                          from the two communities tend to be tuned for different situations:
+                                                                             fuzz testing is typically used in integration testing of complete sys-
+The second argument to QuickCheck.test (on the last line) is the             tems, to detect catastrophic bugs like crash failures and memory
+property: It says, given an integer x and a BST t, that insert t x           unsafety, while PBT is used to flush out logical errors in smaller
+should return a BST. The first argument to QuickCheck.test is a              software modules.
+random data generator, which here generates a pair of both an int
+and an arbitrary BST (obtained by generating an arbitrary binary
+                                                                             3     METHODOLOGY
+tree using tree_gen and using filter to discard trees that are not
+valid BSTs). QuickCheck.test randomly generates hundreds or                  To address the research questions described in §1, we conducted an
+thousands of pairs (x, t) and invoke the property to check each              interview study of developers who use PBT in their work. This study
+pair. If this check ever fails, we have discovered a bug in insert.          was conducted in accordance with the ACM SIGSOFT Empirical
+   Generators like tree_gen are usually written using an embedded            Standards for qualitative surveys.
+domain specific language (eDSL) provided by the PBT framework,
+which includes base generators like int_gen and combinators like             3.1     Population
+both that can be used to create generators for complex data types            As the setting for our study, we chose Jane Street, a financial tech-
+from generators for their parts. The generator language is embedded          nology firm that uses PBT extensively. We recruited 31 participants
+in OCaml (in this case), giving generator writers access to the full         and carried out 30 interviews (one was a joint interview). Partici-
+power of the host language. Generators can require some careful              pants were recruited by a Jane Street developer who volunteered
+tuning to find bugs effectively. This is often because properties have       to coordinate the study: they found instances of PBT in the source
+preconditions that define what it means for an input to be valid. For        tree and contacted the authors of those libraries; announced the
+example, “filter valid_bst tree_gen” is a correct generator of               study on an internal blog; and carried out snowball sampling by
+BSTs: it simply discards trees until tree_gen happens to generate            asking participants for others we should talk to. All participants
+a valid BST. However, this means that the majority of the generated          had some experience with PBT, and most self-reported as having
+Property-Based Testing in Practice                                                                             ICSE 2024, April 2024, Lisbon, Portugal
+
+
+Table 1: Participant backgrounds. Columns 3–5 reflect op-                    Jane Street’s overall financial operations are supported by a di-
+tional questions; participants that didn’t respond to these               verse set of software engineering efforts. We spoke to developers
+are listed at the bottom. ★Measured in years. ★★Stated com-               working on core data structures, distributed systems, compilers,
+fort with PBT on a Likert scale: 7 most comfortable, 1 least.             graphical interfaces, statistical computation, and even custom hard-
+† Participant indicated skepticism of PBT. ‡ Pair interview;
+                                                                          ware. Study participants spanned fifteen teams in four main areas:
+coded as one participant. (At Jane Street’s request, we do not            Trading (2/30, across 2 teams), working directly with traders to
+associate individuals with their teams or company divisions.)             develop technology specific to individual trading desks; Trading
+                                                                          Infrastructure (13/30, across 5 teams), building platforms to sup-
+    ID    Role       SE Exp.★ PBT Exp.★ Comfort★★                         port Jane Street’s trading; Quantitative Research (5/30, across 2
+    1     Tester     12            3                                (6)   teams), writing software to enable research on trading algorithms;
+    3     Maint. 22                17                               (7)   and Developer Infrastructure (11/30, across 6 teams), designing
+    4     Maint. 15                16                               (6)   tools and languages upon which Jane Street’s applications are built.
+    5     Tester     5             7                                (6)   Jane Street developers primarily work in OCaml, a programming
+    6     Tester     16            16                               (7)   language with strong typing, good mechanisms for modularity,
+    9     Tester     4             4                                (5)   and a focus on performance. OCaml is thought of as a functional
+    11    Maint. 10                7                                (6)   language, but it has strong support for imperative programming as
+    14    Tester     8             7                                (5)   well.
+    15    Tester     2             2                                (6)
+    16    Tester     8             8                                (3)   3.2    Protocol
+    18    Tester     1             1                                (5)   We conducted a semi-structured one-hour interview with each
+    20† Tester       5             2                                (5)   participant; Goldstein and Cutler led the interviews. For testers, the
+    23    Tester     6             6                                (5)   prompts were:
+    25‡ Testers 26                 20                               (5)      (1) Tell us about a noteworthy time that you applied PBT.
+    26    Tester     2             2                                (6)         (a) What kinds of properties did you test?
+    28    Tester     7             1                                (5)         (b) How did you generate test inputs?
+    29    Tester     4             5                                (6)         (c) How did you evaluate the effectiveness of your testing?
+      Other Testers: 2, 7, 8, 10, 12, 13, 17, 19† , 21, 22, 24, 27, 30          (d) What did you do to shrink your failing inputs?
+                                                                             (2) Which parts of the PBT process are the most difficult?
+                                                                             (3) What role does PBT play in your development workflow?
+positive experiences. We also explicitly asked for participants who          (4) To whom would you recommend PBT?
+reported neutral or negative feelings about PBT, but they were               (5) In what contexts is PBT most useful?
+difficult to find: most felt they did not have enough experience to          (6) Is there anything that would make PBT more useful to you?
+speak to those feelings. Two self-described PBT-critical developers
+participated in the study. More details about the study participants      The script was designed to attain depth by encouraging reflection
+can be found in Table 1.                                                  on real, memorable experiences with PBT. It evolved somewhat
+   Most of those we interviewed were testers (26/30), who use PBT         over the course of the study, allowing us to validate interesting or
+tools in their day-to-day work, but we also spoke to several main-        unexpected observations from earlier interviews.
+tainers (4/30) who play a role in building and maintaining Jane              A separate script was used with maintainers:
+Street’s PBT infrastructure. These two groups were given different           (1) Have you seen the type of adoption that you want from your
+prompts (see below), reflecting our expectation that the maintain-                PBT tools?
+ers would be able to offer a more “global” perspective, but their            (2) How can QuickCheck be improved?
+responses were coded uniformly since they address the same re-               (3) What do you think it would take to get everyone at Jane
+search questions.                                                                 Street using PBT? Would that be a good thing?
+   Participants who answered an optional background question-                (4) What do you hope we’ll learn from this study?
+naire had between 1 and 26 years of professional Software Engineer-       Time permitting, we also asked maintainers about their use of PBT.
+ing experience (median 7) and between 1 and 20 years using PBT               We did not explicitly interview until saturation; we simply tried
+(median 6). This wide range of experience (for PBT, almost as wide        to recruit a reasonably sized group that was representative of PBT
+as possible, since the first paper on PBT is only 23 years old! [11])     users at the company. However, as we neared the end of the study
+also means we heard from developers at many different points in           we felt we were no longer learning about new aspects or perspec-
+their careers and with differing levels of software development           tives on PBT, and we saw convergence on many of our findings (as
+experience. When asked to rate their comfort with PBT on a Likert         evidenced by the numbers we report in §4). Thus, we do not expect
+scale, these participants were overall quite comfortable with PBT         there are significant holes in our results.
+(median 6 out of 7): all but one were at least “somewhat comfortable”        Once the interviews were complete, they were transcribed using
+(5 out of 7). Working with developers who are already relatively          an automated transcription service [42] and analyzed to extract im-
+fluent users of PBT allowed us to benefit from their well-informed        portant themes following a thematic analysis process [7]. Goldstein
+ideas about how to make PBT better, as well as their frustrations         and Cutler carried out an open coding pass (reading through the
+and challenges.                                                           transcripts and assigning thematic codes as they went). The codes
+ICSE 2024, April 2024, Lisbon, Portugal                            Harrison Goldstein, Joseph W. Cutler, Daniel Dickstein, Benjamin C. Pierce, and Andrew Head
+
+
+we chose focused on participant goals, benefits of PBT, challenges          50,000 places across the code base” to establish reliable behavioral
+encountered, and opportunities for improvement. Once the set of             contracts for library consumers (P13). Another participant used
+codes stabilized, they were validated by a third co-author, then clar-      PBT to check software that interacted with information they were
+ified by the whole team to obtain a final set of clean codes. Finally,      “sending to the [stock] exchange” (P25), which, should it contain
+one co-author performed an axial coding pass (reading through               errors, could incur serious financial costs. An advantage of PBT
+the transcripts and codes again to make connections and ensure              in these cases was its ability to explore edge cases (mentioned by
+consistency) with the updated codebook. Our codebook is available           10/30) since, in the words of one participant, “the bugs are always
+online1 .                                                                   going to happen in cases you didn’t think of” (P21).
+                                                                               For many participants, PBT served to increase confidence that
+3.3     Threats to Validity                                                 their software was robust. They described code tested with PBT as
+While our study covered a wide variety of software teams and tasks,         “solid” (P6, P28), and some (9/30) explicitly mentioned that PBT in-
+it should be noted that participants mostly described using a single        creased their “confidence” that the code was correct. One described
+PBT toolset, in a single language and software development ecosys-          using PBT when they “wanted peace of mind” (P11). Confidence
+tem. Our findings may under-represent the usage patterns and                was accompanied by material evidence of success: a third of partic-
+challenges experienced by those working with other PBT toolsets             ipants (10/30) said their property-based tests found bugs that they
+in other languages. To help us develop findings that generalize             had not found via other methods.
+beyond the Jane Street toolset, we designed our interview protocol             A less obvious benefit of PBT was its ability to help participants
+to focus considerable attention on the conceptual and methodolog-           better understand their work. One participant described properties
+ical aspects of PBT, rather than on details of OCaml’s QuickCheck           as tools that “force [the developer] to think clearly” (P30) about
+implementation or specifics of how it is used at Jane Street. (In the       their code, and another pointed out that sometimes properties fail
+cases where we make observations that are toolset-specific, e.g.,           because the specification (rather than the code) is wrong (P10).
+where we know tools outside of OCaml address some of the prob-              In such cases, failing tests can highlight gaps in the developer’s
+lems we observed, we state this explicitly.) In addition, participants      understanding of the problem space that could lead to issues later.
+were mostly experienced developers who were comfortable with                   Outside of the testing loop, properties were useful for docu-
+PBT. This means that our study under-reports the experiences of             mentation and communication (12/30). One participant noted that
+novice developers. Finally, as discussed above, we did not measure          properties “are part of the interface and part of the documentation”
+saturation as interviews progressed, so there is a chance that more         (P3) and emphasized that, unlike other documentation, properties
+interviews may still have uncovered new insights.                           never fall out of date: whereas a textual description in a comment
+    Another threat comes from the inherent limitations of inter-            might drift from the code’s actual behavior, a property is repeatedly
+view studies. For one thing, participants may not always accurately         re-checked as the code changes. Many also talked about the value
+recall the particulars of their experiences with tools. We did our          of PBT in the code review process (18/30) as a compact way to
+best to mitigate this threat by asking developers to tell us about          express what a particular program does. One remarked: “I feel like
+specific experiences—a standard technique for studies like ours.            [PBT] is very nice for review. . . I really dread seeing 10,000 lines of
+Additionally, our own biases as interviewers may naturally have             tests where you just need to spend hours to read code and under-
+skewed the results; Goldstein and Cutler, who carried out the inter-        stand what’s going on. . . And I think, if there is a QuickCheck test
+views, and Pierce and Head, who also participated with additional           and you look at the property that’s been tested, which is usually
+questions, are all property-based testing researchers who have a            much shorter. . . it gives you much more confidence that the code is
+vested interest in the outcome of the study. Outcomes that show             correct.” (P19)
+PBT in a positive light may have unintentionally been highlighted,             The benefits of PBT were such that even those who were critical
+and criticisms that we have addressed in our prior work may have            of it found it to be beneficial in some situations. During the recruit-
+received extra attention.                                                   ing process we tried to explicitly recruit participants who said they
+                                                                            did not like PBT; we found two (P19 and P20), but the criticism
+4     RESULTS                                                               in their interviews turned out to be constructive suggestions that
+We now present our findings. We start by describing the benefits            were echoed by PBT’s proponents as well, and both primarily spoke
+that PBT delivered to developers and how it fit with the other testing      about situations where PBT was valuable.
+approaches they used. Then we describe in detail their experiences             Moreover, some participants who liked PBT really liked it: one
+writing specifications, designing generators, debugging, and evalu-         said that they “use QuickCheck for everything” (P13) and another
+ating testing success, focusing both on challenges they experienced         asserted that “everyone should be aware of it” (P27). Yet another
+and on opportunities for improving PBT tools and workflows.                 participant argued that “[PBT] is so useful that it’s worth trying
+                                                                            to reorganize your code into the form that it is applicable. PBT is
+4.1     Benefits of PBT                                                     so helpful that if you can reinvent things that way, you should.”
+                                                                            (P10) Several participants (8/30) also talked about evangelizing PBT,
+As might be expected, the primary reason participants used PBT was
+                                                                            encouraging others to use it and increasing its adoption across the
+to assess whether their code was correct. Participants often used
+                                                                            organization.
+PBT to validate widely used or mission-critical code. For example,
+                                                                               We found that this sort of evangelism benefited all parties: PBT
+one participant described using PBT for code that was “used in
+                                                                            becomes more useful as more people on a team or in an organization
+1 https://doi.org/10.5281/zenodo.10407686                                   use it. A culture of supporting PBT can save work: P24 and P27
+Property-Based Testing in Practice                                                                              ICSE 2024, April 2024, Lisbon, Portugal
+
+
+both noted that PBT would be easier to use if more developers in            Expect tests and PBT were both used to test individual software
+the organization provided generators for the types exported by           units during development. PBT was almost always described as
+their modules, making it much easier to test code in other modules       running in Jane Street’s build system, and many developers actu-
+that uses those types. In addition, treating property-based tests as     ally test their properties “locally after each edit” (P2) Participants
+documentation, as many (12/30) did, requires that others in the          described strict time budgets for PBT—no more than “30 seconds”
+organization can read and understand such documentation.                 (P27) and as low as “50 milliseconds” (P11)—to ensure it would not
+                                                                         slow down the build. These time budgets serve to keep PBT from
+                                                                         triggering the 1-minute-per-library timeout that many Jane Street
+                                                                         developers set for their unit test suite (P7).
+4.2     Comparisons to Other Testing Approaches                             Alongside these fast-turnaround unit testing tools, developers
+The most common approach to testing at Jane Street is not PBT,           also used fuzzing tools like AFL [56] and AFL++ [20] for integration
+but rather an example-based unit testing framework called “expect        testing. A third of participants (10/30) mentioned fuzzing, and a few
+tests” [40]. Expect tests are basically conventional example-based       (3/30) described using AFL alongside PBT as part of their testing
+unit tests with editor integration that helps developers create unit     process. In contrast to the strict timeouts set for expect and property-
+tests from the outputs that the system produces. With expect tests,      based tests, one participant described a fuzzer that was forgotten
+developers add code to the system under test that that logs in-          and left running for a year and a half on a spare server (P9). Others
+teresting data to the standard output channel; the expect testing        did not run fuzzing this long, but still considered it to be running
+framework then checks that the output matches the output string          “out of band” (P28), outside of the normal continuous integration
+that was captured from the previous run of the system—it is up           and at time scales on the order of hours or days. The upshot is that
+to the developer to decide which output is intended. Expect tests        fuzzing tools are given much longer to run than PBT tools.
+are integrated into Jane Street’s editor workflow, and they are used
+pervasively in much the same way as frameworks like pytest and
+JUnit are used in other languages. Thus, we can use comparisons          4.3    Writing Specifications
+with expect tests as a proxy for comparisons with example-based          The previous sections have dealt with PBT at a high level; we now
+tests in general where the specifics of expect tests are not relevant.   begin a deeper dive into the specifics of the PBT process and how
+    From a legibility perspective, PBT was sometimes seen as better,     developers described actually using PBT tools. The first step in this
+sometimes worse than expect tests. Expect tests were described as        process is defining one or more properties to test. While participants
+more transparent and “often easier to understand” (P5). P18 said         did describe struggling with this stage, many ultimately developed
+“expect tests. . . explain what they’re doing to a better degree. And    powerful strategies for finding properties.
+it’s easier for someone to come in and review my test,” and P27             Many participants (16/30) said the process of writing specifica-
+made a similar point. But expect tests can sometimes go past from        tions slowed their progress. P4 summed it up like this: “I think the
+“transparent” to just verbose. P17 complained that reading a whole       most common failure mode is actually not knowing what properties
+output string was onerous, and another participant complained            to test.” Challenges ranged from systems that seemed not to have
+“[with expect tests] it’s easy to get into a mode where you just like    properties at all—P1 talked about “a server that serves queries. . . and
+write a test, and you just print out a bunch of crap. And then it’s      has some. . . nuanced behavior” that is not compatible with “logical
+really annoying to like code review that test because there’s just       properties”—to systems whose properties were hard to articulate
+too much stuff” (P7). Properties are more concise.                       in the form of QuickCheck tests: “With [example-based unit tests],
+    PBT and expect tests also present different strengths and weak-      I kind of look at it, I can just make a snap judgment as to whether
+nesses when it comes to test writing. One participant said that          this is okay or not. Trying to formalize that judgment sometimes
+writing an individual expect test is “way easier than writing invari-    can be very difficult.” (P2)
+ants,” but they highlighted that at the scale of a whole test suite         Challenges in articulating properties can come from the way code
+they “love not writing specific unit tests cases” (P13).                 is written. P7 explained that mutable state (e.g., a hidden variable
+    Ultimately there was no universal preference for properties or       that may change between calls to the same function) gets in the way
+expect tests—both should be available in a developer’s toolkit. We       of writing properties: “there’s. . . this hidden state component. . . that
+might infer from participant responses that when code is simple          kind of makes it harder for me to think about what are the right
+enough and examples communicate its behavior well enough, ex-            laws.” P22 also pointed out that “integrating the outside world
+pect tests are an attractively lightweight option. In cases where        and. . . dealing with the interaction of very large and complicated
+the code is particularly difficult to get right or where writing out     systems” makes it less clear how to “meaningfully test with PBT.”
+enough examples becomes tedious, it seems PBT may be a better            (These findings are not surprising. It is well known that code that
+choice.                                                                  interacts with its environment is also a challenge for testing in
+    One area where properties seem to be at a clear advantage over       general, not just PBT: stateful code uses mutable data structures
+expect tests is in the confidence they provide developers. P25 re-       such as hash tables, which might introduce differences between
+marked of Jane Street developers that “[they] go to randomized           runs of the same test if the state is not reset properly; and effectful
+testing when [they’re] not so confident of what [they’ve] done.”         code might rely on external files, databases, or networks, which
+As discussed above, around a third of participants talked directly       may not be safe to access over and over during testing.)
+about PBT building confidence, and the same proportion reported             Despite these difficulties, the developers we spoke to were gener-
+PBT finding bugs that had not been found with other methods.             ally enthusiastic about PBT. One might therefore wonder: Did they
+ICSE 2024, April 2024, Lisbon, Portugal                              Harrison Goldstein, Joseph W. Cutler, Daniel Dickstein, Benjamin C. Pierce, and Andrew Head
+
+
+simply forge ahead regardless, or did they have specific techniques            4.4     Generating Test Data
+for circumventing the difficulties of writing specifications?                  Participants had several goals for generating inputs. First and fore-
+    What we found was that developers often succeeded in apply-                most, many participants (17/30) talked about needing to generate
+ing PBT by being opportunistic in their choice of when to apply                values that satisfy some precondition. This can be extremely impor-
+it. Rather than try to apply PBT to every program at all times,                tant for effective testing: if too few of the generated values satisfy
+participants looked for situations where it offered high leverage:             the property’s precondition, then testing will fail to exercise the
+more confidence for less work. Some participants described this in             code in interesting ways; in the worst case, it may even report false
+general terms as “go[ing] for the low hanging fruit” (P2) or finding           positives. A few of the preconditions mentioned by participants
+places where “the properties are sort of obvious” (P28), but many              are easy to satisfy randomly—for example non-empty strings or
+enumerated specific situations where they would reach for PBT.                 lists—but most are quite hard to satisfy: valid postal addresses, well-
+    Classical Properties (11/30). Certain forms of properties are fa-          structured XML documents, red-black trees, and syntactically valid
+miliar from the PBT literature and from library documentation for              S-expressions are extremely unlikely to be generated by a naïve ran-
+PBT frameworks. These include mathematical properties (e.g., the               dom generator. Participants also described test data requirements
+commutativity of addition), properties drawn from CS theory (e.g.,             going beyond precondition validity. Some (5/30) said they wanted
+repeated sorting has no effect), and properties that naturally fall            their test data to be “realistic”—i.e., similar to the distributions of
+out of a data structure’s invariants (e.g., the ordering condition of a        data the application was likely to see in the real world. Others de-
+BST). These properties may be more accessible because they are                 scribed heuristics for the distribution of the input data, for example
+naturally top-of-mind.                                                         generating lists or trees of a “reasonable size” (P9).
+    Round-Trip Properties (11/30) are also common in the literature;               Participants described a few different ways that they generated
+we heard about them so much more often than the other classical                inputs. Most talked about either handwritten random generators
+cases that they seem worth calling out on their own. These proper-             (19/30), which are the default approach in libraries like QuickCheck,
+ties check that a pair of functions are inverses of one another—for            or derived generators (19/30), which are inferred based on the types
+example, parsing and pretty-printing or encoding and decoding                  of the data to be generated (e.g., the type int list implies a generic
+functions. This situation is easy to notice and easy to test since the         generator for lists of integers).
+properties are incredibly succinct (you call one function, then its                The need for handwritten generators seemed to be a source of
+inverse, and then check that the final result matches the original             friction for many participants. P6 said that writing generators for
+input), so round-trip properties are a popular choice.                         data that satisfies property preconditions is “the biggest annoyance
+    Catastrophic Failure Properties (7/30). Rather than write logical          with trying to use QuickCheck” and many participants thought
+specifications, some participants used PBT to try to provoke cata-             of writing generators as “tedious” (6/30) or “high-effort” (7/30). P2
+strophic failures such as assertion failures and uncaught exceptions.          described the task of writing generators as intruding into their de-
+In general, these kinds of properties provide less confidence in code          velopment process and contributing to a general perception of PBT
+quality (there can still be logical errors, even if the code does not          as “high cost and low value.” Since PBT was usually described as an
+crash), but they help to rule out worst-case scenarios and they are            integral part of the development process, rather than as a separate
+easy to write. (These kinds of specifications are identical to the ones        quality-assurance task, it makes sense that requiring detours into a
+often used for fuzzing; the line between the techniques is blurry,             lengthy generator design process might make PBT less desirable.
+but since the participants were using PBT tools—including complex                  Besides requiring significant effort to use, available tools for writ-
+generators—and testing small units of software, we still consider              ing generators by hand do not easily enable developers to produce
+this relevant for our purposes.)                                               precondition-satisfying values that are also well distributed in the
+    Differential Properties (17/30). A “differential property” (in sense       way they would like. Writing a precondition-satisfying generator
+of differential testing [25]) compares the system under test to a              on its own can be a large task—whole papers have been written
+reference implementation of the same functionality that serves as              about generators for a single complex data type [43]—and account-
+a specification; these are often also called model-based properties            ing for distributional considerations adds even more friction. P13
+(c.f. model-based testing [54]), especially when the reference imple-          said, “there’s a tension in. . . all these handwritten generators be-
+mentation is designed to be an abstract model of the original code.            tween ‘I want kind of coverage of everything’ and ‘I want coverage
+Differential properties were by far the most widely implemented                that is realistic for most inputs.’” As a solution, P13 suggested that
+kind of property. Differential properties were described as a “natural         one might be able to carefully combine two different handwritten
+place to use property based testing” (P3)—indeed, one participant              generators to achieve these competing goals, but thought “that way
+remarked that PBT was challenging in a particular situation in part            lies madness”. Other participants had an idea of the kinds of values
+because a good reference implementation was not available (P1).                they wanted to generate more or less frequently, but they were
+    The common theme of the high-leverage scenarios described                  not sure how to get there: “What should [the probabilities in my
+above is the availability of a succinct abstraction that can be used           generator] look like?. . . I’m sure if I studied probability and statis-
+in writing a property. These PBT scenarios were summed up by                   tics and fully understood how the QuickCheck generating system
+P9 with the following mantra: “[PBT is] most useful when. . . you              worked, I could give better guesses. But all my guesses. . . they’re
+have a really good abstraction with a complicated implementation.”             not educated guesses. They’re just random. . . And that’s a little bit
+We discuss how these high-leverage scenarios should be taken into              of a mental strain.” (P20)
+account to accelerate research in PBT in §5.
+Property-Based Testing in Practice                                                                                ICSE 2024, April 2024, Lisbon, Portugal
+
+
+   In Jane Street’s ecosystem, derived generators are enabled via          intermediate values that were found during shrinking. P1 said “the
+the ppx_quickcheck library, which provides a preprocessor that             shrinker gets it right. . . but [it’s] still useful to see the evolution.”
+runs before the compiler and synthesizes generator code from type
+information. Ideally, this approach is totally automatic, providing a
+generator without any additional user input, and it was described          4.6    Understanding Success
+fondly by participants. Many included it in their workflows (19/30),       Property-based testing not only requires a developer to understand
+and one called it “f***ing amazing” (P5). P13 went so far as to suggest    test failures; it also requires developers to understand whether
+that “you shouldn’t really be handwriting generators, you should           passing a test actually means the software is correct. If the inputs
+be changing the structure of your type [to improve generation].”           provided by the test harness fail to adequately exercise buggy code,
+(For example, if a function being tested takes an int flag but the         a property-based test may pass misleadingly. One participant de-
+only valid values are 0, 1, and 2, then replacing int with an appro-       scribed this situation, recalling a bug in published code that their
+priate enumerated type causes ppx_quickcheck to derive a better            property-based tests failed to catch because the generator was “not
+generator.) Others (7/30) leveraged a combination of derived and           generating the case that [they] had in mind” (P19). P14 worried
+handwritten generators, using derived generators as a foundation           they could not trust their generators because they had “no idea
+on which to build more complex generator programs.                         what it’s generating.” (P14)
+   For both handwritten and derived generators, Jane Street devel-            But, while participants understood that, in principle, tests could
+opers recognized opportunities for tool improvements. P4 described         pass erroneously, 11 of them—more than a third—said they did not
+the process of hand-writing a generator for a mutable data structure       think as hard as they should about testing effectiveness, or whether
+as “overwhelming,” and P6 suggested that libraries could do a better       they had adequately tested their software with their generators.
+job supporting that use case. As for improving PPX-derived genera-         Some (3 of the 11) reported seeing their property catch a couple
+tors, P26 wanted better support for generating values of generalized       of bugs and deciding that they did not have to improve their prop-
+algebraic data types (GADTs), special types in OCaml and some              erties any further; the rest trusted that the generators provided
+other languages that can express fine-grained properties of data.          by OCaml’s QuickCheck library or the ones they derived or wrote
+                                                                           themselves would be good enough. While this group is a minority
+                                                                           of our sample, even a signal of this size is striking: PBT tools such
+4.5     Understanding Failure                                              as derived generators should make it easy for developers to get
+Debugging is often tricky, but participants described ways that            started; they should not (but evidently sometimes do) discourage
+tracking down bugs detected by PBT can be especially so. One               developers from being critical of their test suite.
+participant described their debugging process: after finding a large,         On the other hand, several participants described techniques for
+unwieldy counter-example with PBT, “you have to pull that failure          validating their PBT tests:
+into a separate sort of regression test. . . which runs the same infras-      Mutation Testing (7/30). Some participants intentionally added
+tructure but does it with much more detail. . . Having done that, it       bugs to their code and checked that their tests successfully found
+is still sometimes unclear—like what about this example actually           those bugs. This technique is standard in the testing literature [44,
+causes us to fail?” (P1) This is an instance of a broader problem:         47] and used in testing benchmarks such as Magma [27] and Etna [50].
+random test generators often produce failing examples that are                Example Inspection (8/30). An even simpler way to assess a gen-
+too large to make sense of during debugging. To help developers            erator’s distribution is to look at a handful of examples that the
+understand failing examples, PBT frameworks offer shrinkers that           generator produces; one participant (P26) even designed a small
+transform large inputs into smaller inputs that (presumably) intro-        utility to graphically render one example at a time, to make them
+duce the same bug. One participant in our study deemed shrinking           easier to understand at a glance.
+“necessary” (P15), and two who implemented their own ad-hoc PBT               Code Coverage (2/30). Participants evaluated the code coverage
+frameworks (P8 and P21) insisted that shrinking was one of the             achieved by their tests and used code coverage measurements as
+most important features they implemented.                                  an indication that their properties were thoroughly exploring the
+   That said, participants found it difficult to use the shrinking         space of program behaviors.
+functionality in QuickCheck, which resembles shrinking in many                Property Coverage (1/30). Another participant measured coverage
+PBT frameworks. In such frameworks, shrinking is achieved by               not of the system under test, but of the property itself. This is a
+having users manually write functions that incrementally reduce a          weaker measurement, since it does not say anything about the
+large value (e.g., a string, list, or other more complex data structure)   system under test, but it is much easier to make because it can be
+to a smaller one that triggers the same failure. Participants saw          measured without complex tooling.
+writing shrinkers as an undesirable activity: one plainly stated, “I          A couple of participants (2/30) compensated for gaps in their
+hate writing shrinkers” (P4). One challenge of writing shrinkers           property-based tests by supplementing them with example-based
+was writing them in a way that preserved important non-trivial             unit tests. In these cases, example-based tests were written to test
+invariants: “It’s easy to write a shrinker that accidentally does not      complementary functionality that could not be easily tested with
+preserve some invariant, and that makes your test fail.” (P13) This        properties. As one participant put it, “it’s kind of not a good idea to
+led one participant to ask for a “generic solution [for shrinking],        use QuickCheck in isolation” (P6), as doing so limits the breadth of
+rather than having the user write shrinkers” (P10).                        software behaviors that can be tested.
+   A few participants also described wanting more information                 How might PBT frameworks provide better support these (and
+from the shrinking process—for example, the progressively smaller          other) techniques that give insight into how thoroughly properties
+ICSE 2024, April 2024, Lisbon, Portugal                                Harrison Goldstein, Joseph W. Cutler, Daniel Dickstein, Benjamin C. Pierce, and Andrew Head
+
+
+have been tested? P15 described their ideal interaction with PBT                 part of their testing toolbox, using it to gain confidence when they
+tools, where the tools give “insight into what [PBT] is doing. . . I             were unsure of code’s correctness, especially where correctness
+think a combination of knowing what it’s doing and having more                   was of critical importance. PBT gave them confidence that their
+control of the space that it’s exploring would be interesting.” Partici-         software was operating correctly, finding bugs in code that had not
+pants desired greater awareness of what their tools were doing: P16              been found via other methods.
+called this “visibility”, and P30 called it “inspectability.” Many par-          OB2: Properties are used as devices for communicating specifications.
+ticipants wanted greater visibility to better understand the breadth             Tests are generally a valuable form of documentation, and proper-
+of inputs generated. P18, for instance, desired “visualization of the            ties are no exception. Jane Street developers use them as executable
+test [inputs] being generated”. Others wished to see details such                evidence that code satisfies a specification, which has significant ad-
+as “the answer [of the function under test] on a smaller set of ex-              vantages over traditional documentation that might go stale as the
+amples” (P9), “statistics on. . . edge cases” (P18), statistics describing       code changes. As shown in §4.1 and §4.2, properties were deemed
+generated inputs such as “the average length of [a generated] list”              especially useful for communication during code review: when
+(P25), and code coverage (P9).                                                   properties were available, they were considered a strong signal that
+                                                                                 the code was correct, and when unavailable reviewers sometimes
+4.7     Tradeoffs                                                                asked the submitter to write some. These auxiliary uses for proper-
+As the earlier sections indicate, using PBT is not without costs.                ties are worth keeping in mind for PBT framework designers; for
+From coming up with properties to evaluating testing success, par-               example, some property languages try to make properties easier to
+ticipants found friction in many steps of PBT. Participants described
+seeing themselves as employing PBT more often if its “overhead”
+could be lowered (P26), or if there was less “effort necessary to
+integrate [it] into [their] workflow” (P10).                                                                  Observations
+   Amidst these costs, PBT was often seen as worth the effort. For                    OB1      Property-based testing is being used successfully
+some participants, development of properties was unremarkable.                                 to build confidence in complex systems.
+For others with more involved implementation, it was still worth
+                                                                                      OB2      Properties are used as devices for communicating
+the costs: in the words of P6,“[It was] a huge slog, but I was like
+                                                                                               specifications.
+‘this needs to be right,’ and. . . I’m glad I did it.” P17 describes the
+tradeoffs of using PBT as follows: “doing PBT is both putting in                      OB3      Property-based tests need to be fast; they are
+more effort and saving oneself effort as well.”                                                expected to perform as well as other unit tests.
+                                                                                      OB4      Property-based testing is used opportunistically
+5     LESSONS LEARNED                                                                          in high-leverage scenarios where properties are
+In this section we answer our research questions, setting a course                             readily available; developers rarely go out of
+for upcoming PBT research and tool development that is grounded                                their way to write subtle specifications.
+in findings from the study. Our recommendations cut across the                        OB5      Developers see writing generators as a distrac-
+PBT process, and establish new goals and priorities for different                              tion, preferring to use derived generators.
+areas of PBT research. We first answer RQ1 with observations of                       OB6      Developers may not interrogate properties that
+PBT’s practice that offer a reality check to PBT researchers and tool-                         do not find bugs, even in cases where they ac-
+builders (§5.1), then we answer RQ2 with a technical and empirical                             knowledge they should.
+research agenda motivated by our study (§5.2).
+                                                                                                        Research Opportunities
+5.1     The State of Practice
+                                                                                      RO1      Understand time constraints for property-based
+Recall that RQ1 asks, “What are the characteristics of a successful                            tests.
+and mature PBT culture at a software company?” Based on the
+                                                                                      RO2      Improve support for differential and model-based
+results of our study, we answer: A mature PBT culture considers
+                                                                                               testing.
+PBT a tool to be opportunistically applied in situations of high
+leverage to gain confidence in software and document its behavior;                    RO3      Make more testing scenarios high leverage.
+developers try to keep PBT “out of their way,” expecting it to work                   RO4      Streamline the process of writing well-
+quickly and easily. In this section, we synthesize the results from                            distributed, precondition-satisfying generators.
+the previous section into observations that expand on these ideas.                    RO5      Improve interfaces for shrinking.
+Some observations point forward to §5.2, where we present ideas                       RO6      Improve tools for evaluating testing effective-
+for future research based on our findings.                                                     ness.
+OB1: Property-based testing is being used successfully to build confi-                RO7      Connect evaluation of testing effectiveness and
+dence in complex systems. The broader research community some-                                 generator improvement.
+times situates PBT as a lower-effort, lower-reward alternative to
+formal proofs of correctness, but that perspective underestimates
+PBT as a practical tool for improving software quality. In §4.1, we
+show that the developers at Jane Street found PBT to be a valuable                             Figure 1: Summary of major outcomes.
+Property-Based Testing in Practice                                                                            ICSE 2024, April 2024, Lisbon, Portugal
+
+
+write by providing terse syntax and expressive defaults, but those        Instead, as seen in §4.4, many participants opted to test with gener-
+features may cause problems if they hurt readability.                     ators derived from the types of the data that their programs operate
+                                                                          on.
+OB3: Property-based tests need to be fast; they are expected to perform      This has two implications. First, it means that ongoing work to-
+as well as other unit tests. The PBT literature is not always clear       wards improving generator automation is critical. After all, as easy
+about exactly when in the software engineering process they expect        as current derived generators are to use, they are not always suffi-
+properties to be written, but in §4.2 we observe that PBT’s niche is      cient. The well-liked ppx_quickcheck library, for example, cannot
+testing module-level code during development. This is in contrast         derive generators for properties with complex preconditions. Broad-
+with fuzz testing, which, when used by Jane Street developers, is         ening applicability of derived generators lowers barriers to using
+treated as a separate step and run outside of the standard testing        PBT. Second, it means that developers of manually written genera-
+workflow. This discrepancy makes sense in view of the differing           tor languages must carefully consider any added friction. Languages
+goals that we observed for PBT and fuzzing: PBT is used for module-       that make generators more difficult to write, perhaps because do-
+level tests with often-complex logical specifications, while fuzzing      ing so enables desirable new features, should be cautious—these
+is used for integration-style tests of whole applications, to check       features may be unlikely to see adoption.
+for relatively basic (e.g., assertion failure or uncaught exception)         Separately, languages and tools for writing generators should
+errors. (Look ahead to RO1 in §5.2.)                                      provide a wealth of sensible defaults, and they should optimize the
+   Properties live alongside other unit tests, so they are expected       user experience around common patterns. Participants indicated
+to run as quickly as other unit tests. Knowing this may change            that there was room for improvement in all of these areas, at least
+priorities for some PBT researchers. If users are only testing their      in OCaml’s QuickCheck. (Look ahead to RO4 in §5.2.)
+properties for 50 milliseconds, research advances that increase input
+generation or property execution speed might make the difference          OB6: Developers may not interrogate properties that do not find bugs,
+between bugs being found or not. Conversely, approaches that make         even in cases where they acknowledge they should. A passing prop-
+generators more thorough but significantly slower may not be used         erty sometimes means that a program is free of bugs, but it may also
+unless developers are given reason to accept longer execution times.      mean that the input values are not the right ones to trigger a latent
+                                                                          bug. Developers acknowledged this fact, and they had expectations
+OB4: Property-based testing is used opportunistically in high-leverage    around the kinds of input values that they wanted when testing
+scenarios where properties are readily available; developers rarely go    their properties. (Besides satisfying property preconditions, they
+out of their way to write subtle specifications. Conventional wisdom      wanted them to be realistic, well-distributed in the space, inter-
+in the PBT community sometimes assumes that developers decide             esting enough to cover corner and edge cases, and more.) But, as
+to use PBT and then try to think of a specification, but the study        shown in §4.6, when it came time to decide if their generators met
+participants generally did the opposite: they saw an obvious testable     expectations, developers did not analyze them closely. Developers
+property and then decided to use PBT. We call situations with these       of PBT frameworks, and especially PBT automation, should keep
+readily available properties “high-leverage” testing scenarios.           this in mind: automated tools for generating test inputs risk giving
+   The high-leverage scenarios varied—we are not confident that           developers a false sense of security. They must ensure that their
+we have documented all of the ones available in practice—but a few        tools test thoroughly, because testers may not. (Look ahead to RO6
+are described in §4.3. The most popular was differential or model-        in §5.2.)
+based testing, which compares the code under test to some other
+available implementation. (We are not surprised our participants          5.2    Research Opportunities
+found these techniques useful, after all both differential and model-     Now we tackle RQ2: “What opportunities exist for future work in
+based testing have rich literatures on their own, but we did not          the PBT space, motivated by the needs of real developers?” Based on
+expect to see them so seamlessly incorporated into PBT workflows.)        our results, we conclude: Exciting avenues for PBT research include
+Other high-leverage properties include round-trip properties that         further studies into performance and usability of PBT generators,
+check an inverse relationship between functions and catastrophic          broader and deeper support of high-leverage PBT scenarios, better
+failure properties that make sure a program does not fail completely.     tools for shrinking, and better visibility into the testing process.
+   What does this opportunistic use of PBT mean for researchers           This section discusses research opportunities in software engineer-
+and tool builders? Frameworks can optimize for and automate these         ing, programming languages, and human-computer interaction
+scenarios—tools like Hypothesis Ghostwriter [16] have already             research that will unlock the yet-unrealized potential of PBT. Some
+begun incorporating common PBT scenarios into an automation               research opportunities point backward to §5.1, where we synthesize
+framework—improving the common case and accelerating PBT use              observations that inform these ideas.
+even further. High-leverage scenarios should also be incorporated
+into PBT benchmarks like Etna [50] to make sure that the perfor-          RO1: Understand time constraints for property-based tests. Future
+mance of PBT algorithms is evaluated in a way that reflects their         studies should more thoroughly explore how long developers across
+use in real-world scenarios. (Look ahead to RO2 and RO3 in §5.2.)         the software industry actually budget for running PBT. In this
+                                                                          study, we heard about numbers between 50 milliseconds and 30
+OB5: Developers see writing generators as a distraction, preferring       seconds; that difference is massive, and tools that support PBT
+to use derived generators. Since PBT is often done in the midst of        cannot make optimal decisions around optimization without clearer
+development, developers are reluctant to slow down and write a            data. Furthermore, future research should develop a sense of how
+generator; the task was seen as both difficult and time-consuming.        long is “enough” for common kinds of properties and software so
+ICSE 2024, April 2024, Lisbon, Portugal                            Harrison Goldstein, Joseph W. Cutler, Daniel Dickstein, Benjamin C. Pierce, and Andrew Head
+
+
+tools have the data to argue for larger time budgets where required.         continue to look for ways to use current active-tuning strategies
+Moreover, developers of tools that combine ideas from PBT and                for pre-tuning (e.g., by saving inputs to be run later) and ways to
+fuzzing need to carefully examine their performance and recognize            make pre-tuned generators easier to use (e.g., with interfaces that
+that many PBT users set strict timeouts that may preclude the                help developers write complex generators).
+overhead of measuring code coverage while running tests. This is                 When it comes to precondition-satisfying generators, many seek
+encouraging for tools like Crowbar [17] and HypoFuzz [26] that               to unify languages for writing generators with ones for writing
+allow developers to transition between short-running PBT and                 preconditions. One approach, used in the QuickChick PBT frame-
+longer-running fuzzing with the same properties. (See OB3 in §5.1.)          work [46], uses inductive relations as a language for both properties
+RO2: Improve support for differential and model-based testing. As            and generators [35, 45]. This approach works well in the Coq proof
+the most popular high-leverage scenario for PBT, differential test-          assistant, but complex inductive relations are not expressible in
+ing seems particularly interesting as a topic of further research.           mainstream languages or accessible to non-specialists. Researchers
+Differential testing has a rich literature, as discussed in §6, but          should consider ways to generalize these results. Alternatively, ded-
+in the context of PBT there are still advances within reach. For             icated languages such as ISLa [53] use a common language that
+example, in languages like OCaml with rich module structures,                is closer to the logical connectives one might use in a standard
+researchers should aim to increase automation around differential            programming language; more research should be done to evaluate
+testing and produce a test harness for comparing modules without             if this approach can be applied in common PBT scenarios. Whatever
+requiring any manual setup; Hypothesis Ghostwriter has begun to              dual-purpose language is chosen, this is a compelling path forward.
+incorporate similar ideas with Python’s classes. (See OB4 in §5.1.)          (See OB5 in §5.1.)
+
+RO3: Make more testing scenarios high leverage. Some testing situa-          RO5: Improve interfaces for shrinking. It is clear from the study
+tions are just barely outside the realm of “high-leverage” scenarios,        results that shrinkers would be far more useful if they were both
+and improved tooling could make the difference. For example, P5              more automated and more informative. As we mention above, we
+described a technique that they used to test a poorly abstracted             recommend that existing frameworks improve automation by in-
+module with an overly complicated interface: instead of writing              corporating internal test-case reduction [36, 52], which uses gener-
+normal properties, they instrumented the module with log state-              ators to aid in shrinking, where possible. Internal shrinking has the
+ments, wrote properties about what those logs should look like,              added benefit of always producing valid values, which is difficult
+and then tested the module via an external interface that hides              to achieve otherwise.
+many of its internal details. Generalizing and operationalizing this             Participants also asked to see more intermediate examples from
+technique—e.g., perhaps with temporal logic in the style of Quick-           the shrinking process. Indeed, there are cases where the smallest
+strom [41]—would allow for better testing leverage in cases where            failing example is not the most helpful one: e.g., if the shrinker
+poor abstraction prevents traditional PBT.                                   outputs the tuple (0, 0), one might conclude that any tuple of
+   Other testing situations might be supported better as well. Po-           integers triggers the bug, but it may be that the bug is only found
+tential opportunities include improving PBT support for code with            if the first component is actually 0. This example motivated Hy-
+mutable state (e.g., by saving memory snapshots for repeatable               pothesis to begin developing a tool that will give users a variety of
+tests) and code that interacts with the environment (e.g., via robust        tools for exploring failing tests. Debugging cases using shrinking
+integration with tools for mocking [38]). Increasing the scenarios in        might mean showing many shrunk examples to the user, following
+which PBT works out of the box will naturally make it higher-value           related work in the model-finding literature [14, 18], or even provid-
+for developers. (See OB4 in §5.1.)                                           ing control over the shrinking process (e.g., with novel interactions
+                                                                             allowing the user to click on components of a value to shrink only
+RO4: Streamline the process of writing well-distributed, precondition-
+                                                                             those components).
+satisfying generators. This has been a research goal for the PBT
+community since the beginning. Our study clarifies some promising            RO6: Improve tools for evaluating testing effectiveness. Participants
+paths forward, including improving tools for automated tuning,               reported ad-hoc, piecemeal approaches to understanding their test-
+generalizing unified languages for defining generators alongside             ing effectiveness, and they asked for better ways to visualize test-
+properties, and supporting alternatives to randomness as first-class.        ing feedback. As a simple first step, tools should always announce
+   There are two main options for tuning generator distributions.            counts of discarded test cases (i.e., ones that failed the property’s
+Actively tuned generators modify their distribution live, in re-             precondition) so the developer can catch problems early. Many PBT
+sponse to feedback, whereas pre-tuned generators compute dis-                tools provide some way to aggregate statistics while a property is
+tributions ahead of time. Actively tuned generators are the norm             running (see Haskell QuickCheck’s label and collect functions)
+in the fuzzing literature [20] and have been ported to PBT in many           but OCaml’s QuickCheck hides output when tests succeed, which
+forms [17, 23, 34, 48], but they spend precious testing time on tuning       obscures that information, and more should be done around the
+analysis, making them less useful in time-constrained PBT scenar-            usability and legibility of these kinds of aggregations. Going further,
+ios. Pre-tuned generators can be much faster, but they currently             participants desired (1) better interfaces for scanning through ex-
+require too much programmer effort. For example, reflective genera-          amples of generated values, (2) better ways of visualizing generated
+tors [22] (which build on example-based tuning à la the Inputs from          distributions, and (3) better integration of code and branch cov-
+Hell approach [51]) automate the process of generating realistic             erage information. These could significantly improve developers’
+test inputs, but this automation is only possible if a reflective gen-       understanding of how thoroughly their code has been tested. (See
+erator is already available. Moving forward, the community should            OB6 in §5.1.)
+Property-Based Testing in Practice                                                                              ICSE 2024, April 2024, Lisbon, Portugal
+
+
+RO7: Connect evaluation of testing effectiveness and generator im-        Beller et al. coined the term “Test Guided Development” to describe
+provement. How might a developer clearly express their test data          the strategy that programmers actually use. Our study agrees with
+goals and ensure that the generator achieves them? Future tools           the idea that developers use testing to guide their thinking during
+could tighten the feedback loop wherein a developer evaluates and         development (§4.1). We also corroborate challenges that another
+then expresses how to improve their generator at the same time.           study found around integration testing: Greiler et al. [24] found
+For instance, a future user interface might display a generator’s         that while unit testing is common, integration testing is difficult
+distribution in some graphical form like a bar chart and allow the        and often left to the the software’s users; in our study, developers
+developer to directly manipulate the distribution by dragging bars        struggled to use PBT for integration testing because it is difficult to
+up and down. Alternatively, the developer might be able to click          write specifications of entire systems’ behaviors. (§4.3).
+on those annotations to request that the distribution try harder to          Our study also suggests that PBT addresses testing difficulties
+cover a particular line or balance a particular branch, in the style      raised in the literature. Aniche et al. [1] observed that developers
+of AFLGo [10]. In an ideal world, an insufficient generator could be      try to write relatively “random” test cases, with the hope of acciden-
+caught and fixed in a few clicks, without allowing bugs slip by.          tally stumbling on bugs; this is related to developers’ goals for PBT
+                                                                          generators, discussed in §4.4, although PBT has the advantage of au-
+                                                                          tomating this process in many cases. Another study [25], focusing
+6    RELATED WORK                                                         on differential testing, found PBT to be a valuable tool in a devel-
+We focus, in this section, on related work exploring the usability of     oper’s toolbox, providing a coherence check for important code;
+testing and formal methods tools. Prior work on PBT, testing, and         however, they also found some problems with differential testing
+formal methods usability has made important observations about            systems, such as naïve sampling algorithms that failed to trigger
+the challenges of specification and bug finding, but it has lacked the    bugs and large counter-examples that were difficult to reason about.
+depth and domain focus to paint a clear picture of PBT, its usage,        Our study suggests new tools may address these problems.
+and opportunities for improvement.                                           Formal methods. PBT is sometimes described as a “lightweight
+   Property-based testing. As a precursor to this study, our group did    formal method.” Indeed, a recent position paper [49] argued that
+a smaller-scale pilot study with developers using Hypothesis [21].        testing techniques like PBT and fuzzing were important steps on
+The full-scale study is far more in-depth, and presents more detailed     the way towards more formal verification. Formal methods as a
+and nuanced findings, although talking to Python developers did           field have seen similar calls for usability improvements. A report
+raise a few concerns that were less prevalent at Jane Street, espe-       out of the Naval Research Lab [28] says, “to be useful to software
+cially when it comes to difficulty coming up with specifications.         practitioners, most of whom lack advanced mathematical training
+   A study analyzing open-source libraries using Hypothesis [12]          and theorem proving skills, current formal methods need a number
+evaluated the kinds of properties that developers test in practice,       of additional attributes, including more user-friendly notations,
+and found significant overlap with our “high-leverage” testing sce-       completely automatic (i.e., pushbutton) analysis, and useful, easy
+narios. In particular, they found that both round-trip and differential   to understand feedback.” This report was published in 1998, but, as
+or model-based testing are overrepresented in real-world tests.           our study shows, some of their usability criteria are still not ade-
+   An experience report from Amazon [9] described differential test-      quately met by modern PBT tools. A more contemporary account
+ing as a major use-case of PBT and reported that developers used          agrees that “the user experience of formal methods tools has largely
+shrinking tools for debugging and mutation testing to ensure test-        been understudied” [33] and calls for better education and tools for
+ing effectiveness. Our study confirms these patterns and explores         writing and understanding specifications.
+further PBT use-cases (§4.3), insights around shrinking (§4.5), and
+concerns about testing effectiveness (§4.6).                              7   CONCLUSION
+   Other studies highlight a more narrow set of specific challenges
+                                                                          Our study reveals that, even after two decades of active exploration—
+faced by developers using PBT. One experience report describing
+                                                                          and, increasingly, exploitation—of PBT, there is still much to learn
+PBT use at DropBox [31] cited usability problems when testing
+                                                                          about how it is being used and what challenges it faces in practice.
+timing-dependent code. The report found that sequences of timed
+                                                                          We contribute a wealth of observations about PBT’s use in an
+operations resist shrinking, often remaining unwieldy, and that
+                                                                          industrial setting, along with well-founded ideas for future research
+timing dependence caused tests to be flaky. An education-focused
+                                                                          in PBT that that we, with the help of the broader community, hope
+study using PBT [55] observed that the PBT community lacks good
+                                                                          to pursue in the coming years.
+motivating examples. While the former concern only appeared in
+passing in our study (when discussing PBT’s handling of stateful
+software), the latter—a dearth of good motivating examples—might          ACKNOWLEDGMENTS
+be ameliorated by our characterization of high-leverage testing           We would like to thank John Hughes, Hila Peleg, Zac Hatfield-
+scenarios in §5.                                                          Dodds, and Shriram Krishnamurthi for their input and feedback
+   Testing more generally. Beyond PBT, there is considerable work         on drafts of this paper. Also, thank you to Ron Minsky and others
+studying usability software of testing in general. Our study sheds        at Jane Street for being open to this kind of collaboration. We ap-
+light into how these common testing issues manifest in PBT specif-        preciate the support of the University of Pennsylvania’s PLClub
+ically. Two studies of developers who use IDEs [5, 6] conclude that       and Penn HCI. This work was financially supported by NSF awards
+testing, especially Test Driven Development, is not as prevalent          #1421243, Random Testing for Language Design and #1521523, Expe-
+as conventional wisdom would suggest. Based on these studies,             ditions in Computing: The Science of Deep Specification.
+ICSE 2024, April 2024, Lisbon, Portugal                                             Harrison Goldstein, Joseph W. Cutler, Daniel Dickstein, Benjamin C. Pierce, and Andrew Head
+
+
+REFERENCES                                                                                    [20] Andrea Fioraldi, Dominik Maier, Heiko Eißfeldt, and Marc Heuse. 2020. {AFL++}
+ [1] Maurício Aniche, Christoph Treude, and Andy Zaidman. 2022. How Developers                     : Combining Incremental Steps of Fuzzing Research. https://www.usenix.org/
+     Engineer Test Cases: An Observational Study. IEEE Transactions on Software                    conference/woot20/presentation/fioraldi
+     Engineering 48, 12 (Dec. 2022), 4925–4946. https://doi.org/10.1109/TSE.2021.             [21] Harrison Goldstein, Joseph W Cutler, Adam Stein, Benjamin C Pierce, and Andrew
+     3129889                                                                                       Head. 2022. Some Problems with Properties, Vol. 1. https://harrisongoldste.in/
+ [2] Thomas Arts, John Hughes, Joakim Johansson, and Ulf Wiger. 2006. Testing tele-                papers/hatra2022.pdf
+     coms software with quviq QuickCheck. In Proceedings of the 2006 ACM SIGPLAN              [22] Harrison Goldstein, Samantha Frohlich, Meng Wang, and Benjamin C. Pierce.
+     workshop on Erlang (ERLANG ’06). Association for Computing Machinery, New                     2023. Reflecting on Random Generation. In Proceedings of ACM Programming
+     York, NY, USA, 2–10. https://doi.org/10.1145/1159789.1159792                                  Languages. Seattle, WA, USA. https://doi.org/10.1145/3607842
+ [3] Thomas Arts, John Hughes, Ulf Norell, and Hans Svensson. 2015. Testing AU-               [23] Harrison Goldstein and Benjamin C. Pierce. 2022. Parsing Randomness. Pro-
+     TOSAR software with QuickCheck. In 2015 IEEE Eighth International Confer-                     ceedings of the ACM on Programming Languages 6, OOPSLA2 (Oct. 2022), 128:89–
+     ence on Software Testing, Verification and Validation Workshops (ICSTW). 1–4.                 128:113. https://doi.org/10.1145/3563291
+     https://doi.org/10.1109/ICSTW.2015.7107466                                               [24] Michaela Greiler, Arie van Deursen, and Margaret-Anne Storey. 2012. Test confes-
+ [4] Karen Barrett-Wilt. 2021. The trials and tribulations of academic publishing – and            sions: A study of testing practices for plug-in systems. In 2012 34th International
+     Fuzz Testing. https://www.cs.wisc.edu/2021/01/14/the-trials-and-tribulations-                 Conference on Software Engineering (ICSE). 244–254. https://doi.org/10.1109/
+     of-academic-publishing-and-fuzz-testing/                                                      ICSE.2012.6227189 ISSN: 1558-1225.
+ [5] Moritz Beller, Georgios Gousios, Annibale Panichella, Sebastian Proksch, Sven            [25] Muhammad Ali Gulzar, Yongkang Zhu, and Xiaofeng Han. 2019. Perception and
+     Amann, and Andy Zaidman. 2019. Developer Testing in the IDE: Patterns, Beliefs,               Practices of Differential Testing. In 2019 IEEE/ACM 41st International Conference
+     and Behavior. IEEE Transactions on Software Engineering 45, 3 (March 2019),                   on Software Engineering: Software Engineering in Practice (ICSE-SEIP). 71–80.
+     261–284. https://doi.org/10.1109/TSE.2017.2776152                                             https://doi.org/10.1109/ICSE-SEIP.2019.00016
+ [6] Moritz Beller, Georgios Gousios, Annibale Panichella, and Andy Zaidman. 2015.            [26] Zac Hatfield Dodds. 2023. HypoFuzz. https://hypofuzz.com/
+     When, how, and why developers (do not) test in their IDEs. In Proceedings of             [27] Ahmad Hazimeh, Adrian Herrera, and Mathias Payer. 2021. Magma: A Ground-
+     the 2015 10th Joint Meeting on Foundations of Software Engineering (ESEC/FSE                  Truth Fuzzing Benchmark. Proceedings of the ACM on Measurement and Analysis
+     2015). Association for Computing Machinery, New York, NY, USA, 179–190.                       of Computing Systems 4, 3 (June 2021), 49:1–49:29. https://doi.org/10.1145/3428334
+     https://doi.org/10.1145/2786805.2786843                                                  [28] Constance Heitmeyer. 1998. On the Need for Practical Formal Methods. Technical
+ [7] Ann Blandford, Dominic Furniss, and Stephann Makri. 2016. Analysing Data.                     Report. https://apps.dtic.mil/sti/citations/ADA465485 Section: Technical Reports.
+     In Qualitative HCI Research: Going Behind the Scenes, Ann Blandford, Dominic             [29] John Hughes. 2007. QuickCheck Testing for Fun and Profit. In Practical Aspects of
+     Furniss, and Stephann Makri (Eds.). Springer International Publishing, Cham,                  Declarative Languages (Lecture Notes in Computer Science), Michael Hanus (Ed.).
+     51–60. https://doi.org/10.1007/978-3-031-02217-3_5                                            Springer, Berlin, Heidelberg, 1–32. https://doi.org/10.1007/978-3-540-69611-7_1
+ [8] James Bornholt, Rajeev Joshi, Vytautas Astrauskas, Brendan Cully, Bern-                  [30] John Hughes. 2016. Experiences with QuickCheck: Testing the Hard Stuff and
+     hard Kragl, Seth Markle, Kyle Sauri, Drew Schleit, Grant Slatton, Serdar                      Staying Sane. In A List of Successes That Can Change the World: Essays Dedicated
+     Tasiran, Jacob Van Geffen, and Andrew Warfield. 2021. Using lightweight                       to Philip Wadler on the Occasion of His 60th Birthday, Sam Lindley, Conor McBride,
+     formal methods to validate a key-value storage node in Amazon S3. In SOSP                     Phil Trinder, and Don Sannella (Eds.). Springer International Publishing, Cham,
+     2021.     https://www.amazon.science/publications/using-lightweight-formal-                   169–186. https://doi.org/10.1007/978-3-319-30936-1_9
+     methods-to-validate-a-key-value-storage-node-in-amazon-s3                                [31] John Hughes, Benjamin C. Pierce, Thomas Arts, and Ulf Norell. 2016. Mysteries
+ [9] James Bornholt, Rajeev Joshi, Vytautas Astrauskas, Brendan Cully, Bernhard                    of DropBox: Property-Based Testing of a Distributed Synchronization Service. In
+     Kragl, Seth Markle, Kyle Sauri, Drew Schleit, Grant Slatton, Serdar Tasiran, Jacob            2016 IEEE International Conference on Software Testing, Verification and Validation
+     Van Geffen, and Andrew Warfield. 2021. Using Lightweight Formal Methods                       (ICST). 135–145. https://doi.org/10.1109/ICST.2016.37
+     to Validate a Key-Value Storage Node in Amazon S3. In Proceedings of the ACM             [32] JetBrains. 2021. Python Developers Survey 2021 Results. https://lp.jetbrains.com/
+     SIGOPS 28th Symposium on Operating Systems Principles (SOSP ’21). Association                 python-developers-survey-2021/
+     for Computing Machinery, New York, NY, USA, 836–850. https://doi.org/10.1145/            [33] Shriram Krishnamurthi and Tim Nelson. 2019. The Human in Formal Methods. In
+     3477132.3483540                                                                               Formal Methods – The Next 30 Years (Lecture Notes in Computer Science), Maurice H.
+[10] Marcel Böhme, Van-Thuan Pham, Manh-Dung Nguyen, and Abhik Roychoud-                           ter Beek, Annabelle McIver, and José N. Oliveira (Eds.). Springer International
+     hury. 2017. Directed Greybox Fuzzing. In Proceedings of the 2017 ACM SIGSAC                   Publishing, Cham, 3–10. https://doi.org/10.1007/978-3-030-30942-8_1
+     Conference on Computer and Communications Security (CCS ’17). Association for            [34] Leonidas Lampropoulos, Michael Hicks, and Benjamin C. Pierce. 2019. Coverage
+     Computing Machinery, New York, NY, USA, 2329–2344. https://doi.org/10.1145/                   guided, property based testing. PACMPL 3, OOPSLA (2019), 181:1–181:29. https://
+     3133956.3134020 event-place: Dallas, Texas, USA.                                              doi.org/10.1145/3360607
+[11] Koen Claessen and John Hughes. 2000. QuickCheck: A Lightweight Tool for                  [35] Leonidas Lampropoulos, Zoe Paraskevopoulou, and Benjamin C. Pierce. 2017.
+     Random Testing of Haskell Programs. In Proceedings of the Fifth ACM SIGPLAN                   Generating good generators for inductive relations. Proceedings of the ACM on
+     International Conference on Functional Programming (ICFP ’00), Montreal, Canada,              Programming Languages 2, POPL (2017), 1–30. https://dl.acm.org/doi/10.1145/
+     September 18-21, 2000, Martin Odersky and Philip Wadler (Eds.). ACM, Montreal,                3158133 Publisher: ACM New York, NY, USA.
+     Canada, 268–279. https://doi.org/10.1145/351240.351266                                   [36] David R. MacIver and Alastair F. Donaldson. 2020. Test-Case Reduction via Test-
+[12] Arthur Corgozinho, Marco Valente, and Henrique Rocha. 2023. How Developers                    Case Generation: Insights from the Hypothesis Reducer (Tool Insights Paper). In
+     Implement Property-Based Tests. In Conference: 39th International Conference on               34th European Conference on Object-Oriented Programming (ECOOP 2020) (Leibniz
+     Software Maintenance and Evolution (ICSME 2023).                                              International Proceedings in Informatics (LIPIcs), Vol. 166), Robert Hirschfeld and
+[13] Ermira Daka and Gordon Fraser. 2014. A Survey on Unit Testing Practices and                   Tobias Pape (Eds.). Schloss Dagstuhl–Leibniz-Zentrum für Informatik, Dagstuhl,
+     Problems. In 2014 IEEE 25th International Symposium on Software Reliability                   Germany, 13:1–13:27. https://doi.org/10.4230/LIPIcs.ECOOP.2020.13 ISSN: 1868-
+     Engineering. 201–211. https://doi.org/10.1109/ISSRE.2014.11 ISSN: 2332-6549.                  8969.
+[14] Natasha Danas, Tim Nelson, Lane Harrison, Shriram Krishnamurthi, and Daniel J.           [37] David R MacIver, Zac Hatfield-Dodds, and others. 2019. Hypothesis: A new
+     Dougherty. 2017. User Studies of Principled Model Finder Output. In Software                  approach to property-based testing. Journal of Open Source Software 4, 43 (2019),
+     Engineering and Formal Methods (Lecture Notes in Computer Science), Alessandro                1891. https://joss.theoj.org/papers/10.21105/joss.01891.pdf
+     Cimatti and Marjan Sirjani (Eds.). Springer International Publishing, Cham, 168–         [38] Tim Mackinnon, Steve Freeman, and Philip Craig. 2000. Endo-testing: unit testing
+     184. https://doi.org/10.1007/978-3-319-66197-1_11                                             with mock objects. Extreme programming examined (2000), 287–301.
+[15] Zac Hatfield Dodds. 2022. current maintainer of Hypothesis (https://github.com/          [39] Barton P. Miller, Lars Fredriksen, and Bryan So. 1990. An Empirical Study
+     HypothesisWorks/hypothesis). Personal communication.                                          of the Reliability of UNIX Utilities. Commun. ACM 33, 12 (dec 1990), 32–44.
+[16] Zac Hatfield Dodds and David R. MacIver. 2023. Ghostwriting tests for you —                   https://doi.org/10.1145/96267.96279
+     Hypothesis 6.82.0 documentation. https://hypothesis.readthedocs.io/en/latest/            [40] Minsky. 2015. Testing with expectations. https://blog.janestreet.com/testing-
+     ghostwriter.html                                                                              with-expectations/
+[17] Stephen Dolan and Mindy Preston. 2017. Testing with crowbar. In OCaml Work-              [41] Liam O’Connor and Oskar Wickström. 2022. Quickstrom: property-based accep-
+     shop.                                                                                         tance testing with LTL specifications. In Proceedings of the 43rd ACM SIGPLAN
+[18] Tristan Dyer, Tim Nelson, Kathi Fisler, and Shriram Krishnamurthi. 2022. Apply-               International Conference on Programming Language Design and Implementation
+     ing cognitive principles to model-finding output: the positive value of negative              (PLDI 2022). Association for Computing Machinery, New York, NY, USA, 1025–
+     information. Proceedings of the ACM on Programming Languages 6, OOPSLA1                       1038. https://doi.org/10.1145/3519939.3523728
+     (April 2022), 79:1–79:29. https://doi.org/10.1145/3527323                                [42] Otter.ai. 2023. Otter.ai - Voice Meeting Notes & Real-time Transcription. https://
+[19] Carl Eastlund. 2015. Quickcheck for Core.             https://blog.janestreet.com/            otter.ai/
+     quickcheck-for-core/                                                                     [43] Michał H. Pałka, Koen Claessen, Alejandro Russo, and John Hughes. 2011. Testing
+                                                                                                   an Optimising Compiler by Generating Random Lambda Terms. In Proceedings
+                                                                                                   of the 6th International Workshop on Automation of Software Test (AST ’11). ACM,
+Property-Based Testing in Practice                                                                                                   ICSE 2024, April 2024, Lisbon, Portugal
+
+
+     New York, NY, USA, 91–97. https://doi.org/10.1145/1982595.1982615 event-place:          where they are. http://arxiv.org/abs/2010.16345 arXiv:2010.16345 [cs].
+     Waikiki, Honolulu, HI, USA.                                                        [50] Jessica Shi, Alperen Keles, Harrison Goldstein, Benjamin C Pierce, and Leonidas
+[44] M. Papadakis, M. Kintis, J. Zhang, Y. Jia, Y. L. Traon, and M. Harman. 2018.            Lampropoulos. 2023. Etna: An Evaluation Platform for Property-Based Testing
+     Mutation Testing Advances: An Analysis and Survey. Advances in Computers                (Experience Report). Proc. ACM Program. Lang. 7 (2023). https://doi.org/10.1145/
+     (Jan. 2018). http://dx.doi.org/10.1016/bs.adcom.2018.03.015                             3607860
+[45] Zoe Paraskevopoulou, Aaron Eline, and Leonidas Lampropoulos. 2022. Comput-         [51] Ezekiel Soremekun, Esteban Pavese, Nikolas Havrikov, Lars Grunske, and An-
+     ing correctly with inductive relations. In Proceedings of the 43rd ACM SIGPLAN          dreas Zeller. 2020. Inputs from Hell: Learning Input Distributions for Grammar-
+     International Conference on Programming Language Design and Implementation              Based Test Generation. IEEE Transactions on Software Engineering (2020).
+     (PLDI 2022). Association for Computing Machinery, New York, NY, USA, 966–980.           https://doi.org/10.1109/TSE.2020.3013716 Publisher: IEEE.
+     https://doi.org/10.1145/3519939.3523707                                            [52] Jacob Stanley. 2017. Hedgehog will eat all your bugs. https://hedgehog.qa/
+[46] Zoe Paraskevopoulou, Cătălin Hriţcu, Maxime Dénès, Leonidas Lampropoulos,          [53] Dominic Steinhöfel and Andreas Zeller. 2022. Input invariants. In Proceedings of
+     and Benjamin C. Pierce. 2015. Foundational Property-Based Testing. In Inter-            the 30th ACM Joint European Software Engineering Conference and Symposium
+     active Theorem Proving (Lecture Notes in Computer Science), Christian Urban             on the Foundations of Software Engineering (ESEC/FSE 2022). Association for
+     and Xingyuan Zhang (Eds.). Springer International Publishing, Cham, 325–343.            Computing Machinery, New York, NY, USA, 583–594. https://doi.org/10.1145/
+     https://doi.org/10.1007/978-3-319-22102-1_22                                            3540250.3549139
+[47] Goran Petrovic and Marko Ivankovic. 2018. State of Mutation Testing at Google.     [54] Mark Utting and Bruno Legeard. 2010. Practical Model-Based Testing: A Tools
+     In Proceedings of the 40th International Conference on Software Engineering 2017        Approach. Elsevier.
+     (SEIP).                                                                            [55] John Wrenn, Tim Nelson, and Shriram Krishnamurthi. 2021. Using Relational
+[48] Sameer Reddy, Caroline Lemieux, Rohan Padhye, and Koushik Sen. 2020. Quickly            Problems to Teach Property-Based Testing. The art science and engineering of
+     generating diverse valid test inputs with reinforcement learning. In Proceedings        programming 5, 2 (Jan. 2021). https://doi.org/10.22152/programming-journal.org/
+     of the ACM/IEEE 42nd International Conference on Software Engineering (ICSE             2021/5/9
+     ’20). Association for Computing Machinery, New York, NY, USA, 1410–1421.           [56] Michał Zalewski. 2022. American Fuzzy Lop (AFL). https://github.com/google/
+     https://doi.org/10.1145/3377811.3380399                                                 AFL original-date: 2019-07-25T16:50:06Z.
+[49] Alastair Reid, Luke Church, Shaked Flur, Sarah de Haas, Maritza Johnson, and
+     Ben Laurie. 2020. Towards making formal methods normal: meeting developers
+

packages/opencode/specs/simulation-research/text/2026-evolution-python-tests-to-pbt.txt

@@ -0,0 +1,928 @@
+            On the Evolution of Python Test Cases into
+                      Property-based Tests
+                   Cindy Wauters                           Ruben Opdebeeck                                  Coen De Roover
+             Software Languages Lab                    Software Languages Lab                          Software Languages Lab
+             Vrije Universiteit Brussel                Vrije Universiteit Brussel                      Vrije Universiteit Brussel
+                Brussels, Belgium                         Brussels, Belgium                               Brussels, Belgium
+            cindy.suzy.wauters@vub.be               ruben.denzel.opdebeeck@vub.be                       coen.de.roover@vub.be
+
+
+
+   Abstract—Conventionally, unit tests exercise their unit under       Scala2 , Python3 and JavaScript4 . Proposing domain-specific
+test on predetermined input to validate its behavior. The advent       input generation strategies and invariant properties, research
+of property-based testing frameworks has led to unit tests that        initiatives have brought property-based testing to challenging
+exercise the unit under test on randomly-generated inputs, for
+which the unit’s behavior has to satisfy developer-specified invari-   application domains such as telecom software [2], distributed
+ant properties. The promise of increased coverage and stronger         synchronization services [19], neural networks [33], and cyber-
+validation may lead developers to evolve existing conventional         physical systems [13].
+unit tests into property-based ones. This paper reports on the            Despite these successes, property-based tests often consti-
+results of an empirical study of 257 property-based unit tests         tute only a fraction of the test suite. According to a 2023
+(PBT) from 64 open-source repositories that have evolved from a
+conventional unit test. The study examines each PBT-introducing        JetBrains survey [22], the most common PBT framework
+commit for change patterns, and for the type of features of            for Python, Hypothesis, was only used by 5% of Python
+property-based testing frameworks that are adopted. Next, it           developers. To bring the benefits of property-based testing
+investigates the impact of PBT adoption by running test cases to       to more projects, researchers have proposed tool support for
+compute changes in code coverage and test failures. Finally, the       creating property-based tests [16], [38]. However, there is a
+study investigates the evolution of the newly-introduced PBT by
+comparing its first to its most recent version. The study’s findings   gap in the research when it comes to the evolution of existing
+include that 37 out of 257 evolutions required no changes to the       test cases into property-based tests. Therefore, in this paper, we
+body of the test. When changes are required, these are most            investigate these evolutions, their impact, and the maintenance
+likely to target setup code followed by oracle code. Over half of      of the evolved test cases. By doing so, we aim to provide
+the studied PBTs use off-the-shelf input generators, yet see an        valuable insights for testers, tool builders, and researchers
+improvement in code coverage by an average of 4 statements.
+Additionally, for 5 test cases, the PBT found a counterexample         alike. With this paper, we make the following contributions:
+where the original unit test passed. Finally, 80% of the evolved          • We collect a dataset of 257 test case evolutions across
+test cases are still present in the current version of the project.          64 open-source GitHub repositories written in Python.
+   Index Terms—Empirical study, software testing, property-
+based testing, code coverage                                                 Our data collection targets Python projects as, at the time
+                                                                             of writing, it is one of the most popular programming
+                       I. I NTRODUCTION                                      languages [21]. We provide a replication package [40]
+   Conventionally, unit tests take an example-based form in                  containing the resulting dataset and the implementation
+which the unit under test is exercised on predetermined input.               of all subsequent analyses.
+For a unit test that should be run with several inputs, param-            • An empirical analysis of the changes required to rewrite
+
+eterized unit testing [35] frameworks support substituting the               a conventional unit test into a property-based test, as well
+example input by a parameter for which the developer provides                as whether test cases evolved afterwards.
+a predetermined collection of values. When enumerating these              • An empirical analysis of the impact of these evolutions
+
+input values becomes intractable, property-based testing [6]                 on the system under test in terms of changes in code
+frameworks support generating them randomly according to                     coverage and test failures.
+a developer-specified input generation strategy. For each of              The remainder of this paper is structured as follows. Sec-
+the generated inputs, the property-based test (PBT) will check         tion II provides a motivating example and introduces the
+that the behavior of the unit under test satisfies a developer-        research questions. Section III details the dataset of real-world
+specified invariant property.                                          test evolutions we will be working with. Then, Section IV,
+   The QuickCheck framework [6] for the Haskell program-               Section V, and Section VI present our research questions. For
+ming language was the first to support property-based testing.         each of the three research questions, we detail the research
+Since then, more than 35 property-based testing tools [18]
+have been introduced for various languages, such as Java1 ,              2 https://github.com/typelevel/scalacheck
+                                                                         3 https://github.com/HypothesisWorks/hypothesis
+  1 https://github.com/jqwik-team/jqwik                                  4 https://github.com/dubzzz/fast-check
+def test_encode_decode():                                              However, it can be difficult for developers to come up
+    txt = "hello world"                                             with invariant properties to test [14]. The Hypothesis Quick-
+    assert decode(encode(txt)) == txt                               start [43] hints at existing parameterized unit tests (PUTs) and
+                                                                    existing sources of randomness in a test suite for inspiration.
+Listing 1: An example-based test for encode and decode              However, to our knowledge, no research has looked into real-
+                                                                    world test case evolutions such as the one described above. In
+import pytest                                                       this paper, we answer the following research questions:
+                                                                       • RQ1: How do conventional unit tests evolve into
+testdata = ["hello world", "test -", "(/test)"]
+@pytest.mark.parametrize("txt", testdata)                                 property-based tests? In this research question, we
+def test_encode_decode(txt):                                              investigate real-world commits for the changes required
+    assert decode(encode(txt)) == txt                                     to rewrite a conventional example-based unit test or
+                                                                          parameterized unit test into a property-based one.
+Listing 2: A parameterized unit test for encode and decode             • RQ2: What is the impact of adopting property-based
+                                                                          testing? To investigate the impact of these test case
+                                                                          evolutions, we compute the difference in code coverage
+method, the findings, and present an objective analysis of                before and after the rewrites as well as the test failures.
+these findings. Section VII discusses the actionable insights          • RQ3: How are property-based tests maintained?
+gained from the findings across the research questions. Here              To assess the effort required to maintain the resulting
+we also describe the threats to validity and their mitigation. In         property-based tests, we investigate whether there are
+Section VIII we position our work and compare it to current               changes made to the test cases after their rewriting into
+research into property-based testing.                                     a property-based test, or if they are removed.
+
+ II. M OTIVATING E XAMPLE AND R ESEARCH Q UESTIONS                                           III. DATASET
+                                                                       In order to study real-world unit test evolutions, we first
+   Test cases may evolve over time. To illustrate such an evo-
+                                                                    need a dataset of open-source repositories that contain at least
+lution, consider an encode function and its inverse decode.
+                                                                    one property-based unit test (PBT) that started as either an
+Listing 1 depicts an example-based unit test (EBT) checking
+                                                                    example-based unit test (EBT) or as a parameterized unit test
+that decoding the encoding of "hello world" results in
+                                                                    (PUT). Our data collection will start from Python repositories
+the same string. This unit test validates the behavior of the
+                                                                    that have a dependency on the Hypothesis [27] framework
+unit under test for a single developer-provided example string.
+                                                                    for property-based testing. While other empirical studies have
+   Using Pytest [23], the most common Python testing frame-         collected such repositories before (e.g., [32], [39]), we collect
+work, the example-based unit test from Listing 1 can be gener-      our own to ensure the dataset contains ample instances of test
+alized into a parameterized unit test [36]. Listing 2 depicts an    case evolutions.
+example generalization in which a parameter txt substitutes            1) Data collection: We first use the GitHub API to find
+for the earlier example input. Three possible values have been      open-source Python repositories that have a dependency on
+provided for the parameter, thereby validating the behavior on      Hypothesis. In order to exclude toy projects, we only include
+predetermined strings that contain special characters.              projects that have at least three stars. We also filter out larger
+   Using Hypothesis [26], [27], the most popular property-          projects with more than 10 stars. While these projects can
+based testing framework for Python, the test case can be            be valuable to study, they are also difficult for an outsider
+generalized further. Listing 3 depicts the resulting property-      to understand and reason about. We do not consider those
+based test, specifying that the composition of decoding after       projects suitable for a deep manual investigation, as required
+encoding is the identity operation for all strings generated        for this empirical study section. In total, we considered 1388
+using st.text. Hypothesis will automatically generate mul-          repositories that have a dependency on Hypothesis and satisfy
+tiple strings, each of which will be encoded and decoded and        our inclusion criteria.
+compared to the original string. This way, the property-based          Next, we use PyDriller [34] to find all PBTs in a repository
+unit test might uncover defects for edge cases the developer did    that were once either an EBT or a PUT, and are now a PBT.
+not enumerate in the example-based nor in the parameterized         To this end, we rely on the presence of @given decorators in
+version of the unit test.                                           a file with a PBT. The GitHub API alone does not suffice, as
+                                                                    noted by prior empirical studies into real-world Hypothesis
+from hypothesis import given, strategies as st                      usage (e.g., [8], [32], [39]), due to the existence of other
+                                                                    libraries named Hypothesis. The @given decorator in a test
+@given(st.text())                                                   file, in contrast, denotes the entry point of a Hypothesis-based
+def test_encode_decode(txt):                                        PBT. If none exists, the repository does not contain a PBT, and
+    assert decode(encode(txt)) == txt                               is excluded. If we find at least one PBT in a file, we investigate
+                                                                    all previous versions of the file. In case the PBT exists in the
+ Listing 3: A property-based test for encode and decode             initial commit of the file, PBTs have always existed in that
+                # LOC Python   # evolved test cases   # commits     1      def test_tolerance_sign_EBT():
+    Mean            7575              4.02              923         2          r = R.from_min_max(3, 2)
+    Std Dev.       17147              6.66              1996        3          assert r.tolerance == 0.2
+                                                                    4
+    median          2607                2               294
+                                                                    5
+    1st Quar.       1178                1               121.5       6      @given( unit=st.sampled_from((U, I, P, R)),
+    3rd Quar.       5346                4               680.5       7              a=st.floats(allow_nan=False),
+    Min             164                 1                15         8              b=st.floats(allow_nan=False))
+                                                                    9      def test_tolerance_sign_PBT(unit, a, b):
+    Max            109516              39              10274
+                                                                    10         eu = unit.from_min_max(a, b)
+                                                                               assert eu.tolerance >= 0
+TABLE I: Characteristics of the 64 repositories investigated 11
+in this paper. Lines of Code computed by Cloc [10].
+                                                                         Listing 4: A real test case before and after becoming a PBT.
+                                                                         Some simplifications applied for readability. Case taken from
+                                                                         https://github.com/wese3112/eecalpy
+file, and we do not consider this file relevant for investigating
+test case evolution. In case there is a previous version of the
+file in which the PBT did not exist, while it does in a newer
+                                                                         end, we categorize all changes to the body of the test case into
+version, we manually investigate the last version before the
+                                                                         the following categories (four per type):
+introduction with the first version after the introduction. We
+                                                                            • Changes to the oracle code: non-semantic changes, se-
+perform this step manually to ensure that we also retrieve
+evolutions of tests that have been renamed. Additionally, we                  mantic changes, additional oracles, removal of oracles.
+                                                                            • Changes to the setup code: non-semantic changes, se-
+take note of whether the transformation in question is from an
+EBT to a PBT, or from a PUT to a PBT. We recognize PUTs as                    mantic changes, additional setup code, removal of setup
+unit tests adorned with the @pytest.mark.parametrize                          code.
+decorator. This renders their recognition purely syntactic.                 To delimit test oracle code, we follow the definition of Barr
+   2) Characteristics of the dataset: Out of the 1388                    et al. [3]. It is the part of the code that decides whether
+Hypothesis-dependent GitHub repositories initially discovered,           the system under test behaves as expected. This does not
+575 have at least one PBT present. Upon filtering for test case          necessarily mean that only (nor all) the code in an assert
+evolutions, we identified 64 repositories that feature at least          statement is part of the oracle code. Code that defines the
+one evolution from an EBT or PUT into a PBT —amounting                   oracle is included too.
+to 257 PBT before-after mappings. For each PBT, we have                     As setup code, we consider those parts of the body of
+a mapping from the test before the PBT introduction to the               the test case that create or retrieve values through which the
+test after the PBT introduction. For a minority of PBTs, the             behavior of the system under test will be validated. In other
+test before its introduction was split into multiple PBTs. On            words, it sets up the system under test. This is similar to the
+the other hand, some tests were merged into a single PBT. In             definition of a test fixture by Meszaros [28]. This is sometimes
+these cases, we consider the same PBT multiple times (each               also known as test context. However, we only consider the
+time with a different mapping). Our 257 before-after mappings            code inside the body of the test case.
+therefore consist of 252 unique tests before PBT introduction               As a non-semantic change to the body of a test case,
+and 250 unique PBTs. Of the total 257 test case evolutions,              we consider straightforward refactorings such as changing a
+182 test cases follow the EBT to PBT evolution, whereas 75               constant to a value generated by the input generation strategy.
+follow the PUT to PBT evolution. Table I presents summary                For semantic changes, in contrast, we consider anything that
+statistics of the distribution of the data across the most recent        changes the semantics of a test case (e.g., extra calculations
+version of the repositories. Included are the lines of code in           within existing code). An addition or removal of setup code
+Python (blank lines and comments not included), the number               means any lines added removed, that can be considered setup
+of investigated test case evolutions, and the total number of            code. An addition or removal of oracle code is the addition
+commits per repository.                                                  of, for example, a new assert statement, or the removal of an
+                                                                         old one.
+ IV. RQ1: H OW DO CONVENTIONAL UNIT TESTS EVOLVE                            A test case can evolve into a PBT through several of
+            INTO PROPERTY- BASED TESTS ?                                 these types of changes. To illustrate, consider Listing 4 which
+                                                                         depicts one of the test cases in our dataset before and after
+   RQ1 investigates how PBTs can evolve out of conventional              evolving into a PBT. Line 2, as well as the r.tolerance
+test cases. To this end, we manually investigate the 257 afore-          in line 3 is considered the setup code: these lines define
+mentioned real-world before-after mappings in the dataset.               what will be tested later. The assert statement, excluding
+                                                                         r.tolerance, is oracle code. This test case changes in two
+A. Research Method
+                                                                         different ways. First, a non-semantic change to the setup code:
+   1) Changes to the test case body: First, we consider how              on line 10 and 11, we see a renaming of r to eu. Additionally,
+test cases are updated when they evolve into a PBT. To this              hardcoded values R, 3, and 2 are replaced with the parameters
+that will now be generated data. Second, a semantic change           •   Control: assume, @note, @event, target, used in
+in the oracle code: as the values are not hardcoded anymore,             the test case to modify how it is treated by the test runner,
+an exact expected value is not known anymore, and the oracle             or to further document it.
+needs to be updated. The change from == to >= makes this a            3) Input generation strategies: As the input generation
+semantic change.                                                   strategy is an important part of every PBT, we also categorize
+   Furthermore, it is also possible that the test case changed     the strategies chosen at migration time as follows:
+beyond recognition when it was rewritten into a PBT. During           • Simple: strategies that generate standard values (e.g.,
+our manual investigation, we mark such test cases as replaced,           integers, floats, strings), and collections thereof (e.g., lists,
+rather than updated, if we cannot recognize it within the PBT            tuples). Strategies that compose standard values into a
+anymore. Some large-scale studies into test evolution have               user-defined data type are classified in the next category
+used thresholds on automatically-computed similarity metrics             instead.
+instead (e.g., 66% used in [5] and [11]), but these make less         • Custom, with reuse: strategies that are custom to the
+sense when applied to small test cases instead of large files.           project, but reused across multiple test cases.
+We therefore rely on human judgement of the labelers instead.         • Custom, no reuse: strategies that are custom to the
+If the labelers cannot identify similar code in the test case            project, but not reused across multiple test cases
+before and after the evolution, and especially in their assert        • Other: usage of third-party libraries for input generation.
+statements, the test is tagged as different test cases.                  For example, to generate NumPy values.
+   The labeling was performed individually by the first and           We assign each test case one of these four categories.
+second authors of this paper. After a first round of labeling,     Custom strategies encompass cases where developers define
+we assessed inter-rater agreement using Cohen’s Kappa [7]          their own, complex, input generators for the test cases. We also
+calculated for each category, obtaining a mean Kappa of 0.475,     consider any usage of builds and from_type in this cate-
+signifying “moderate agreement” [25]. To avoid systematic          gory, as developers use them to generate custom user-defined
+disagreements, the two labelers discussed and resolved dis-        data types. When a test generates both values of standard and
+agreements on a random sample of 20% of the entire dataset to      user-defined data types, we consider its generation strategy
+establish a shared understanding. For instance, a recurring dis-   custom overall.
+agreement was the presence of setup code in assert statements.
+After resolving these differences, the labelers individually       B. Results
+performed a second round of labeling on the remaining 80%,            1) Changes to the test case body: We first discuss the
+resulting in a mean Kappa of 0.727, signifying “substantial        changes that conventional unit tests undergo when rewritten
+agreement” [25]. The remaining disagreements were discussed        into a property-based test. Figure 1 shows the results for both
+among the two labelers to obtain consensus.                        EBTs and PUTs that evolved into a PBT.
+   2) Features of the property-based testing framework that           For test cases that evolved from EBTs in our dataset, it is
+are adopted: Property-based testing frameworks often offer         especially common (117/182) for the setup code to be changed
+features to customize a test case. The @example decorator of       in a non-semantic way (for example, replacing a hardcoded
+Hypothesis, for example, enables specifying that a value needs     value to instead refer to the input generation strategy, as
+to be tested on every run of the test case. This can be used       shown in Listing 4). The second most common change is for
+with known edge cases that the developer is aware of. The          the oracle code to be updated with a non-semantic change.
+@settings decorator lets developers specify test budgets,          Other common changes are additions of oracle code, semantic
+the maximum number of examples to be generated, etc. It is         changes to oracle code, and removal of oracle code or setup
+also possible to reproduce test failures by re-using the random    code. Additionally, 12% of cases changed beyond recognition.
+input generation seed that found the counterexample (e.g., the        Tests that evolve from PUTs are a bit different. Here,
+input for which the property did not hold).                        we see fewer changes in the test cases overall. The most
+   We track the use of these features at the time the PBT was      common change is the addition of oracle code, followed by
+introduced. By doing so, we aim to discover whether devel-         the addition of setup code, and semantic changes to the oracle
+opers are using them from the onset. We use the following          code. Notably, 35/75 test case evolutions from PUTs in our
+categorization according to the Hypothesis API [42]:               dataset required no change at all in the body of the test case
+   • Strategies: @given to generate inputs. The next para-         (in comparison to 2 stemming from EBTs). Many PUTs might
+     graph categorizes the input strategies specified through      already be testing a property, especially if they contain no
+     this decorator further.                                       hardcoded expected values in the parameter. This means that
+   • Explicit inputs: @example, which enable the tester to         evolving them into a property-based test can be less labor-
+     provide inputs to test each run.                              intensive for developers.
+   • Reproducing inputs: @reproduce_failure and                       2) Features of the property-based testing framework that
+     @seed, to reproduce the data from previous test               are adopted: The results are shown in Table II. All tests
+     failures.                                                     specified at least one input generation strategy, as expected
+   • Settings: @settings, to impose test budgets, a maxi-          from a PBT. Hypothesis itself sees this decorator as the
+     mum number of examples to generate, etc.                      entry point of a PBT. Control (specifically assume), was
+                                                                                                                                                             64.3%                                   PBT origin
+                                                                                                                                                                                                  from example-based test
+                     60                                                                                                                                                                           from parameterized test
+
+
+
+                     50
+  Percent of tests
+
+
+
+
+                     40
+
+
+
+                     30                                            29.1%
+
+
+                                                                                                                         22.5%                                                      22.5%
+                                    20.0%    20.3%
+                     20
+                          17.6%                                                                    17.0%
+                                                          16.0%                                                                  16.0%
+                                                                                                                                                     14.7%
+                                                                                                                                                                      12.0%                              12.1%
+                     10                                                         8.0%
+                                                                                                             6.7%                        6.6%                                                   6.7%                6.7%
+
+
+                     0
+                                                                                        tic
+
+
+
+
+                                                                                                                                                                              tic
+                                                             tic
+
+
+
+
+                                                                                                                                                                                                                          e
+                                                                                                               e
+
+
+
+
+                                                                                                                                                       tic
+
+
+
+
+                                                                                                                                                                                                  e
+                                    dd
+
+
+
+
+                                                                                                                                   d
+                                                                                                             ov
+
+
+
+
+                                                                                                                                                                                                                         as
+                                                                                                                                                                                                ov
+                                                                                                                                 ad
+                                                                                     an
+
+
+
+
+                                                                                                                                                                           an
+                                                          an
+
+
+
+
+                                                                                                                                                     an
+                                  :a
+
+
+
+
+                                                                                                            em
+
+
+
+
+                                                                                                                                                                                                                    tc
+                                                                                                                                                                                            em
+                                                                                   m
+
+
+
+
+                                                                                                                                                                      em
+                                                       em
+
+
+
+
+                                                                                                                                                 em
+                                                                                                                               p:
+                               le
+
+
+
+
+                                                                                                                                                                                                                     s
+                                                                                se
+
+
+
+
+                                                                                                        :r
+
+
+
+
+                                                                                                                            t-u
+
+
+
+
+                                                                                                                                                                                            r
+                               c
+
+
+
+
+                                                                                                                                                                                                                  te
+                                                                                                                                                                       s
+                                                   :s
+
+
+
+
+                                                                                                                                                 s
+
+
+
+
+                                                                                                                                                                                         p:
+                            ra
+
+
+
+
+                                                                               -
+
+
+
+
+                                                                                                       le
+
+
+
+
+                                                                                                                                                                    n-
+                                                                                                                          Se
+                                                                            on
+
+
+
+
+                                                                                                                                                                                                                t
+                                                                                                                                              p:
+
+
+
+
+                                                                                                                                                                                      t-u
+                           O
+
+
+
+
+                                                  le
+
+
+
+
+                                                                                                                                                                                                             en
+                                                                                                       c
+
+
+
+
+                                                                                                                                                                  no
+                                                                                                                                           t-u
+                                                                                                    ra
+                                                                           :n
+                                                  c
+
+
+
+
+                                                                                                                                                                                    Se
+
+
+
+
+                                                                                                                                                                                                             er
+                                               ra
+
+
+
+
+                                                                                                   O
+
+
+
+
+                                                                                                                                                                 p:
+                                                                                                                                         Se
+                                                                       le
+
+
+
+
+                                                                                                                                                                                                          iff
+                                              O
+
+
+
+
+                                                                                                                                                              t-u
+                                                                      c
+
+
+
+
+                                                                                                                                                                                                         D
+                                                                   ra
+
+
+
+
+                                                                                                                                                             Se
+                                                                   O
+
+
+
+
+Fig. 1: The changes example-based (blue) and parameterized (orange) unit tests undergo when evolving into a property-based
+test. Each test case can undergo multiple changes from different categories.
+
+
+                                                  From EBT                                    From PUT                      novice users of the framework, or because the test cases did
+ Feature                                    # of test cases            %           # of test cases                 %        not yet have the time to evolve and be tailored to the project.
+ Strategies                                       182              100                        75                 100        Therefore, we also investigate how the evolved test cases are
+ Explicit inputs                                      4            2.20                       2                  2.67       maintained in Section VI-B2.
+ Reproducing inputs                                   0                0                      0                    0           3) Input generation strategies: Table III depicts the results
+ Settings                                          11              6.04                       8                  10.67      for our analysis of the input generation strategies used. The
+ Control                                              2            1.10                       14                 18.67      majority of PBTs that evolve from an EBT have a simple input
+                                                                                                                            generation strategy for standard data types. However, for 76 of
+TABLE II: Features initially adopted from PBT framework.                                                                    the 120 simple cases (a majority), more refined configurations
+                                                                                                                            are used of these simple input generators (such as min/max
+                                                                                                                            bounds for integers). 60 of the EBTs that evolved into PBTs
+the second most common feature, followed by @settings                                                                       now have a custom input generation, of which 24 are reused
+(mostly imposing deadlines on the test cases) and explicit                                                                  multiple times across different test cases.
+inputs (@example). The fact that we see no PBT using                                                                           When considering PBTs that have evolved from a PUT, just
+failure-reproducing features is not surprising, as it is the first                                                          over half of the PBTs have a simple input generation strategy.
+PBT-version of the test cases. They might not yet have run                                                                  Of those 43 PBTs, 35 refined configurations. All custom input
+enough to want to reproduce a specific failure-inducing input.                                                              generation strategies are reused across multiple test cases. It is
+   Compared to test cases that evolved from EBTs, test cases                                                                common for evolved PUTs from the same project to already
+that evolved from PUTs are more likely to adopt these features                                                              share the same input parameter before their evolution into a
+of the PBT framework. One reason for this could be that                                                                     PBT. In our dataset, the use of libraries for input generation is
+the developers are already more familiar with more advanced                                                                 not common (only two instances). The only used library was
+testing frameworks.                                                                                                         the Hypothesis NumPy extension5 .
+   Corgozinho et al. [8] also performed a preliminary study                                                                    Several test cases already contained some form of random-
+into commonly used features across 86 PBTs. Their dataset                                                                   ization before being rewritten into a PBT. Afterwards, this
+does not consider the initial PBT introduction, and contains                                                                randomization is still reflected in the input generation strategy.
+more well-established, big projects that make use of property-                                                              For example, if a test case previously contained a statement a
+based testing. In their dataset, they observe a higher feature
+adoption rate compared to ours. This could be because devel-                                                                  5 https://hypothesis.readthedocs.io/en/latest/reference/strategies.html#
+opers evolving a PBT from an existing test might be more                                                                    hypothesis-numpy
+                                From EBT      From PUT                 that define behavior in their test files, these should indeed
+                                #      %      #      %                 be considered. For others, including test files can lead to a
+           Custom, with reuse   36    19.78   32   42.67               wrong conclusion when the test cases themselves have shrunk
+           Custom, no reuse     24    13.19   0      0                 or increased in size. We therefore report both on the absolute
+           Simple               120   65.94   43   57.33               and relative coverage twice, once with the test files included
+           Other                 2    1.10    0      0                 and once with the test files excluded.
+                                                                          We were able to run 219 of the test cases before PBT-
+TABLE III: Input generation strategies used within PBTs                introduction, and 219 test cases after PBT-introduction. How-
+evolved from an EBT or a PUT.                                          ever, not all were able to obtain code coverage. In the end,
+                                                                       we were able to run and obtain code coverage for 211 test
+                                                                       case evolutions before and after the introduction of the PBT
+= np.random.randint(1, 10), a strategy of the test                     (meaning a total of 422 test cases). To ensure the test cases ran,
+case would become @given(st.int(1, 10)).                               some minimal changes to the code were needed. For example,
+                                                                       one test case contained a broken import. Upon correcting this
+  RQ1 Non-semantic changes to the setup code are es-                   import, the test case did run. However, we made sure to not
+  pecially common in evolutions stemming from EBTs                     change the semantics of the system under test. In some cases,
+  (117/182). For evolutions from PUTs, it is more common               tests failed but still resulted in some code coverage. Their
+  to require no changes in the body of the test case at all            results are also included.
+  (35/75). The addition of settings is the most common
+                                                                          3) Test failures: Code coverage is not the only way to deter-
+  feature (19/257) early on in the adoption of the PBT
+                                                                       mine whether tests are effective. During the data collection of
+  framework. Most test cases use simple input generation
+                                                                       Section V-A2, not all test cases passed, and some test oracles
+  strategies (163/257).
+                                                                       threw an assertion error. For some test cases, this happened
+                                                                       before their evolution into PBT, whereas for others, the PBT
+       V. RQ2: W HAT IS THE IMPACT OF ADOPTING
+                                                                       did not pass. We investigate how often this occurs, and the
+              PROPERTY- BASED TESTING ?
+                                                                       failed assertions in each.
+  With research question two, we aim to find the impact
+of introducing PBTs. We consider commit messages, code                 B. Results
+coverage, and test failures of the 257 test evolutions.                   1) PBT-introducing commits: First, we report on our anal-
+                                                                       ysis of the commit messages of the PBT-introducing commits.
+A. Research Method                                                        Only eight commit messages mention anything positive or
+   1) PBT-introducing commits: We investigate and report on            negative about Hypothesis or property-based testing (beyond
+73 commit messages across 64 repositories that evolve at least         simply stating that they have started using it). One commit
+one conventional test case into a PBT. We aim to uncover               message mentions improving code coverage to find more
+reasons as to why the developer decided to evolve their test           errors across the code. Two commit messages mention the new
+case, and their potential opinions on this change.                     PBTs failing (a bug was found, but not yet fixed). Finally, five
+   2) Code coverage: We first explore whether code coverage            commit messages use the words better, trickier, or improved.
+improves once a conventional test case has evolved into a                 Four commit messages also mentioned bug fixes of some
+PBT. Improving coverage can be a goal for some PBT-                    sort, meaning the PBT was introduced at the same time as a
+introducing developers, even though it does not necessarily            bug fix. This can indicate that the PBT at introduction might
+improve test suite effectiveness [20]. Moreover, research has          have found a bug that the initial conventional test case did
+proposed variants of PBT with that explicit aim (i.e., coverage-       not, after which the developer fixed the bug. Alternatively, the
+guided PBT [24], [29], [30]).                                          developer may also be introducing bug fixes and PBTs at the
+   We run each test case with statement coverage before                same time to ensure their most recent bug fix is tested more
+and after the introduction of Hypothesis while measuring               in the future. We did not find any evidence in the commit
+coverage using Coverage.py [4], a well-established Python              message supporting either hypothesis, but Section V-B3 looks
+library for test coverage. We only compute the coverage for            into test case failures before and after the commit.
+each individual test case, instead of for the entire test suite as a      2) Code coverage: To verify whether there is a difference
+whole. For relative coverage (i.e., in percentage), we normalize       in coverage before and after the evolution into PBT, we use a
+the executed statements by those in the files executed by the          Wilcoxon signed-rank test [41] as our data is non-parametric
+test case rather than by those in the entire project. We opted         and paired (code coverage of a test case before and after the
+for this design as coverage across all statements of the entire        PBT-introducing commit). We consider a p-value of <0.05
+project would be very low for individual test cases. We also           statistically significant. The test results are listed in Table IV.
+report on coverage in absolute numbers. This can be useful             For all 4 cases, we observe a statistically significant difference.
+for test cases that lead to the execution of several files. Note          First, we look at the change in number of statements
+that, by default, coverage.py considers all executed Python            covered. Figure 2 shows box plots for the distribution of the
+files including the files that define the test case. For projects      absolute statement coverage before and after the test case
+             Stmt (all)            % (all)                Stmt (no test)         % (no test)      coverage ∆   # Stmt (all)   % (all)   # Stmt (no test)   % (no test)
+ p-value       0.006           2.497 × 10−20             1.477 × 10−13               0.045        Mean           +13.09       +2.31          +4.01            -0.82
+                                                                                                  Std Dev.        27.79        6.06          16.81             5.5
+TABLE IV: p-values of change in coverage, in terms of
+                                                                                                  median           +11        +0.57           +0               +0
+absolute and relative statement coverage.
+                                                                                                  1st Quar.        +2         -0.06           +0              -1.26
+                  total # ran tests        # passing          # failing      with coverage        3rd Quar.        +17        +2.46           +3             +0.28
+ Before PBT                  219                  199             20                211           Min              -78        -10.09          -75            -21.90
+ After PBT                   219                  207             12                211           Max             +223        +32.61          +77            +32.52
+
+TABLE V: Number of passing and failing test cases. We could                                      TABLE VI: Changes in code coverage, both in terms of
+not obtain coverage for all tests (as shown by the last column).                                 number of statements covered and change in the coverage
+                                                                                                 percentage, for both all files and all files without test files
+                                                                                                 included.
+
+    2500
+                                                                                                 evolution, taking into account all executed files (denoted “all”
+                                                                                                 in the plot) and all executed files without test files (denoted
+    2000                                                                                         “no test”). While we see a slight improvement in median
+                                                                                                 and quartiles when all executed files are taken into account,
+    1500
+                                                                                                 the change becomes much more subtle when excluding the
+                                                                                                 executed test files. One reason could be that property-based
+                                                                                                 test cases execute more lines of test code. Even if the tests
+    1000
+                                                                                                 themselves are not necessarily longer, the inclusion of input
+                                                                                                 generators can add more test code to cover —especially for
+     500                                                                                         projects that have defined custom ones in test files.
+                                                                                                    Figure 3 on the other hand shows the relative statement
+                                                                                                 coverage in percentage. As mentioned before, by default,
+       0
+                                                                                                 coverage.py only takes into consideration files that have at
+              Before (all)          After (all)          Before (no test)     After (no test)
+                                                                                                 least one executed line of code, meaning files that are never
+                                                                                                 covered by the test case are not considered. Here we also see
+                                                                                                 an improvement when taking into account test files, but once
+Fig. 2: Statements covered by test cases before and after PBT-
+                                                                                                 they are removed from the data, we notice a slight decrease
+introduction. The first two plots show coverage across all files,
+                                                                                                 in code coverage on average.
+the second two without test files included.
+                                                                                                    Table VI shows the deltas for each of the four measure-
+                                                                                                 ments. While the mean absolute number of covered statements
+                                                                                                 without test files included does improve slightly (4.01), the
+                                                                                                 relative coverage percentage goes down slightly. This could
+     100                                                                                         be due to extra files being discovered, or more code being
+                                                                                                 introduced during the PBT-introducing commit. Most notice-
+      80                                                                                         ably, one project’s covered statements decreases by 75. Upon
+                                                                                                 further investigation, the test case in question contained many
+                                                                                                 assert statements. While the conventional test case before
+      60
+                                                                                                 PBT-introduction passed, Hypothesis found a counterexample
+                                                                                                 on the first assert, thereby stopping the test case and not
+      40                                                                                         executing the subsequent assert statements (leading to a drastic
+                                                                                                 decrease in coverage). Additionally, 26 test cases from the
+                                                                                                 same repository all had one additional change when evolved
+      20
+                                                                                                 into PBT: the removal of a call to the logging function (which
+                                                                                                 was three statements), leading to a decrease of coverage of
+       0                                                                                         indeed 3 statements for these 26 test cases.
+             Before % (all)        After % (all)        Before % (no test)   After % (no test)      3) Test failures: Table V shows the number of tests failed
+                                                                                                 before and after PBT-introduction. Two test cases did not pass
+                                                                                                 as a conventional unit test, nor after having been rewritten into
+Fig. 3: Coverage percentage by test cases before and after                                       a PBT. For 18 test cases, the initial conventional test did not
+PBT-introduction. The first two plots show coverage across                                       pass, but the updated PBT did pass. In another ten cases, the
+all files, the second two without test files included.                                           initial test passed but the resulting PBT did not.
+   In terms of the before, one project with one evolution              •   Updates to the test case encompassing both semantic and
+had a test case that resulted in an assertion error. The PBT-              non-semantic changes.
+introducing commit also contained a bug fix. Another project            We make one exception for 39 test cases originating from
+had three failing test cases that returned a type error. After the   one repository. In the most recent version of that repository,
+introduction of the PBT, all three passed. However, the PBT-         most Python code has been removed, and is scheduled to be
+introducing commit does not introduce a bug fix (meaning             added at a later stage. Because of this drastic rewrite affecting
+the bug was in the test code itself). Another project had two        the latest version in the repository, we instead consider its most
+failing test cases, with an attribute error and a type error. For    recent version with the test cases still present.
+both, the PBT versions did pass. In one of the bigger projects          Additionally, we report on the survival rate of the test cases
+in our dataset, with 16 evolved test cases, ten tests did not        by looking at the number of commits since introduction, or
+pass before the evolution. In three cases this was because of        the number of commits between introduction and removal.
+an assertion error (with the other seven not passing due to a           2) Are more parts of the PBT framework adopted?: As de-
+value not being defined). All ten of these test cases passed after   velopers become more acquainted with property-based testing
+being updated to a PBT. These results suggest that people are        in their projects, they might discover features unbeknownst
+introducing PBTs when fixing known errors, perhaps to ensure         to them before. Therefore, similar to Section IV-A2, we
+more thorough testing of those parts of the system.                  investigate whether the most recent version of their PBT
+   When looking at the failing tests cases after the introduction    uses more features of the PBT framework. We use the same
+of PBTs, we found five test cases for which Hypothesis               categories of features from the Hypothesis API [42] to classify
+reported a counterexample for which the original test case           the additions. As all PBTs already had an input generation
+passed. Further, as mentioned in Section V-B1, some commits          strategy when they were introduced, we investigate whether
+did contain additional bug fixes. This indicates that PBTs can       this strategy has been updated in the meantime.
+uncover bugs not found by conventional test cases validating
+the same behavior, thus improving bug detection. For the             B. Results
+other 5 non-passing PBTs, we found other errors (such as                1) Do Property-based Test Cases Change?: To answer this
+ValueErrors), even though the original test case passed before.      question, we compare the most recent commit in the GitHub
+                                                                     repository to the PBT-introducing commit. Table VII shows
+  RQ2 8 PBT-introducing commit messages convey an                    the number of commits the test cases have survived (for those
+  intent to improve test cases to improve coverage or find           that are still present in the current version of their project,
+  bugs. We observed a statistically significant change in            i.e., the number of commits between introduction and the
+  statement coverage, with on average an increase of 4               most current commit), or how many repository commits they
+  statements. We also found evidence of PBTs being intro-            survived before their deletion. Most test cases in our dataset
+  duced along with a fix for a bug in the code, and of PBTs          (110) are unchanged. Their corresponding repositories have
+  finding counterexamples where the original corresponding           also enjoyed the fewest commits since the PBT-introducing
+  conventional test passed.                                          one (with 99 on average, with a standard deviation of 205).
+                                                                     Interestingly, one test case survived 1728 repository commits
+       VI. RQ3: H OW ARE PROPERTY- BASED TESTS                       without undergoing any changes itself. A total of 90 test cases
+                    MAINTAINED ?
+                                                                     did undergo changes since their introduction, both semantics-
+   We consider the most recent version of the 250 unique PBTs        preserving and semantics-affecting ones. We observe that these
+from the in total 257 test case evolutions, and investigate the      test cases also survived more commits, with on average 642
+changes (or removals) they have been subject to since.               commits and a higher standard deviation. This is expected as
+                                                                     PBTs need to co-evolve with their system under test.
+A. Research Method
+                                                                        Finally, 50 PBTs were removed from the projects since
+   For each test case, we compare the version resulting from         their initial introduction. These test cases also survived fewer
+the PBT-introducing commit to the version in the most recent         commits; 352 on average. However, one test case survived
+commit in its repository. We consider the following:                 3573 commits to the repository before being removed. Reasons
+   1) Do Property-based Test Cases Change?: We investigate           cited for the deletion of these test cases were removal of the
+whether the 250 PBTs in our dataset change over time, for            functions they tested (5), unexpected behavior (3), and the tests
+instance in response to changes in the system under test. For        being slow (1).
+each PBT, we manually compare against the same test case in             2) Are more parts of the PBT framework adopted?: Finally,
+the most recent version of the GitHub repository. For each of        we consider the changes made to PBTs in terms of the parts of
+our test cases, we consider the following:                           the PBT framework that are adopted. Table VIII describes the
+   • No changes to the test case. The rest of the project and        results. In 65 test cases, the input generation strategies were
+     other test cases might have changed, but the investigated       updated after their introduction. One repository in particular
+     PBT has not.                                                    mentions doing so to improve the running time of the test
+   • Removal of the test case. In this case, we also investigate     cases. Only one repository added an explicit input to test for on
+     possible reasons why.                                           each run. Additionally, one test case added a seed decorator
+                        No changes       Changes       Removal       easier to evolve unit tests that exercise the unit under test on
+           Mean             99            642              352       inputs of standard data types (e.g., integers) into a PBT with
+           Std Dev.       205.31         1450.79        967.80       such a strategy. Our findings should motivate practitioners to
+           median           51            146               34       prioritize their generalization efforts on those unit tests first.
+           1st Quar.        48              64              17          Finally, a study by Goldstein et al. [14] reports that PBT
+           3rd Quar.        63            146               71       adopters have difficulty identifying the properties to test for.
+           Min               1              17              7        Using existing test cases, rather than creating new ones, might
+           Max             1728           5886           3573
+                                                                     help. This is especially true for parameterized unit tests (PUT),
+                                                                     and especially those that do not list expected outputs for every
+TABLE VII: Number of commits since the introduction of the           given input, as those might already be testing a property (but
+PBT. Data split for the test cases that did not change at all, for   not yet randomly generating input). Indeed, for 35/75 PBTs
+those with changes, and for those removed in the meantime.           that evolved from a PUT (Section IV-B), no changes to the
+                                                                     body of the test case were required at all. This suggests that
+           Feature                               # of test cases     PUTs can be considered a stepping stone towards, or even an
+           Change in strategies                        65            intermediate step in a planned introduction of PBTs.
+           Addition of explicit inputs                 1                For tool developers: Recent years have seen the introduc-
+           Addition of reproducing inputs              1             tion of tool support for creating PBTs. From Hypothesis’
+           Addition of settings                        19            own ghostwriter6 , over using LLMs to write PBTs [13], [38],
+           Addition of control                         0             to tools that aim to aggregate similar test cases into one
+           Addition of exceptions                      0             PBT [31]. Tools that take EBTs and transform them into PUTs
+                                                                     also exist [37]. However, there remains a need for tools that
+TABLE VIII: Changes in used PBT features in the most recent          support developers in evolving existing, dissimilar, test cases
+versus the initial PBT-introducing commit of the test case.          into PBTs. Our findings for RQ1 (Section IV-B), and our
+                                                                     supporting dataset, can inspire tool builders with examples of
+to reproduce inputs. 19 of the test cases eventually got some        real-world transformations that can be partially automated.
+form of settings added, which doubles their total number.               As shown in Section IV-B2, not all features of the property-
+                                                                     based testing framework are immediately adopted in newly-
+  RQ3 Most PBTs (200/250) in our dataset have survived               evolved PBTs. In Section VI-B2 we found that features such
+  since their introduction, with 90 having gone through              as PBT settings are often included only later in the life of
+  changes. Removed test cases survived on average 352                a PBT. We also saw that inside some PBTs, developers were
+  commits to the repository. 65 test cases saw a change              using if-statements, where an assume might have been more
+  in their input generation strategy since their introduction.       appropriate. Therefore, future tools could not only help with
+  Finally, 19 test cases received additional settings.               the evolution of test cases into PBTs, but also aid with the
+                                                                     introduction of features to optimize existing PBTs.
+                        VII. D ISCUSSION                                For researchers: There are still few empirical work that
+   We now distil the actionable insights from our findings, for      compares conventional unit tests to PBTs. Ours is, to our
+software testers, tool builders, and researchers alike. We also      knowledge, the first to do so in a one-to-one comparison of test
+discuss the threats to the validity of our findings.                 cases before and after their evolution into property-based ones.
+                                                                     Our comparison on test failures and code coverage has been
+A. Actionable insights                                               insightful, and could inspire other one-to-one comparisons on
+   For software testers: As evidenced by our findings for            other metrics. For instance, mutation score (recently used in
+RQ2 (Section V-B), evolving well-chosen unit tests into              prior work [32], see Section VIII) has not yet been explored
+property-based ones can increase their code coverage. We             in a one-to-one comparative setting.
+moreover found evidence of such real-world evolutions finding
+                                                                     B. Threats to validity
+counterexamples where the original test passed without any
+failures, leading to a deeper validation of the system under            Following standard practice in empirical software engineer-
+test. We even found instances of these evolutions being con-         ing, we discuss potential threats to the validity of our results.
+ducted alongside bug fixes, most likely to test for regressions      For each threat, we describe the applied mitigation strategies.
+more thoroughly. Our findings should motivate practitioners             Construct validity: We considered test case evolutions that
+to consider unit tests that have missed edge cases in the past       are found in open-source GitHub repositories, which can
+as candidates to be evolved into property-based ones.                raise quality concerns. We mitigated this threat by applying
+   Prior studies have found that implementing input generation       strict inclusion criteria, resulting in 575 high-quality candidate
+strategies for a PBT can be tedious and effort-intensive [14].       repositories with at least one PBT. As most of those PBTs did
+Nonetheless, our findings for RQ1 (Section IV-B) indicate            not result from an evolution, we were left with fewer test case
+that many evolved PBTs in our dataset use but a simple,                6 https://hypothesis.readthedocs.io/en/latest/reference/integrations.html#
+built-in input generation strategy. Practitioners may find it        ghostwriter
+evolutions to study, but the study subjects still stem from 64             Goldstein et al. [14] interviewed 30 developers from a
+different repositories, ensuring diversity.                             fintech company, to find insights about how people experience
+   Further, while false negatives cannot be excluded, all test          PBT in practice. Another paper by Goldstein et al. [15] reports
+case evolutions were manually inspected by both the first               that developers experience difficulties in finding properties to
+and the second author, ensuring the dataset is free of false            test for. Our research looks at open-source projects rather than
+positives. The same goes for the before-after mappings for              the industry, and investigates existing test cases.
+each test case evolution, which were created by the first author           A study by Wauters and De Roover [39] looked into the use
+and inspected and confirmed by the second author.                       of PBTs by 28 open-source machine learning projects. The
+   Manual labeling of test case evolutions was also used to             study investigated common positive and negative sentiments
+answer RQ1. We mitigated the threat of subjective bias by               expressed in GitHub commits and code comments, which
+having multiple labelers and multiple labeling rounds until             parts of a project were tested, as well as the complexity of
+a substantial inter-rater agreement was reached. To answer              data generation strategies. Our study also investigates data
+RQ2, we used tools to execute test cases and compute their              generation strategies but does not focus on ML-intensive
+code coverage. To mitigate the threat of measurement bias               projects that often require more complex data.
+introduced by the tooling, we only used mature open-source                 Ravi and Coblenz [32] recently performed a large-scale
+tools that are well-established within the community.                   empirical study on PBTs in 426 Python programs. Their study
+   Internal validity: For RQ2, we measure changes in code               uses a data flow analysis to automatically categorize real-world
+coverage induced by the evolution into a PBT. However, the              PBTs into 12 categories. Additionally, they used mutation
+introduction of a PBT is not always the only change in a                testing to investigate how many mutations a project’s existing
+commit. If additional changes are present, the total number             PBTs can find compared to the project’s existing conventional
+of statements may change, which can confound the observed               tests. Our research looks at the impact of evolving test cases,
+coverage differences and lead to incorrect conclusions. To              and compares a test case before its evolution into a PBT to
+mitigate this threat, we manually inspected commits exhibiting          the resulting PBT. This means we compare two test cases that
+large coverage changes. In addition, we report both absolute            are equivalent in terms of the behavior they are supposed to
+and relative coverage, computed with and without the test files         validate, and we do so one-to-one.
+included, as test files are more likely to change due to the               Unlike our study, the aforementioned papers do not inves-
+introduction of PBTs.                                                   tigate the evolution of PBTs, but rather consider snapshots of
+   External validity: Finally, our study focuses on Python              PBTs at one point in time.
+test cases that use the Hypothesis framework. Hypothesis                            IX. C ONCLUSION AND F UTURE W ORK
+is the most widely recognized PBT framework for Python,                    This paper studied the evolution of property-based unit tests
+and Python is ranked among the most popular programming                 from conventional ones, the impact of such PBT-introducing
+languages. Nevertheless, our findings may not generalize to             evolutions, and how the resulting PBTs are maintained after-
+other programming languages nor to other PBT frameworks.                wards. We found that the evolution of parameterized unit tests
+                                                                        into a PBT commonly requires no updates to their test body
+                       VIII. R ELATED W ORK
+                                                                        at all. For example-driven unit tests, in contrast, the evolution
+   Several studies have investigated the introduction and main-         commonly required non-semantic changes to the test body.
+tenance of various types of test cases, from tests in general [1],      These account for the introduction and usage of a randomly
+[9], [44] over GUI tests [5] to performance tests [11]. A               generated input value. The input generation strategy of PBTs
+few studies have also looked into changes in code coverage              that evolved from existing unit tests is often simple. Impact-
+induced by software patches, e.g., [12] and [17]. However,              wise, we found that code coverage can improve when unit tests
+our work is the first to study how PBTs can evolve from                 are replaced by PBTs. More importantly, we found evidence
+conventional unit tests, and to study the impact on code                of bugs found through or alongside the introduction of these
+coverage —rather than changes in code coverage over time                PBTs. Most of the PBTs in our dataset are still present in the
+or induced by changes to the system under test.                         most recent version, with almost half having gone through
+   Zooming in on other studies targeting property-based test-           some maintenance during their life time. The adoption of
+ing, Corgozinho et al. [8] sampled 86 PBTs from GitHub, and             additional PBT features is not uncommon either.
+categorized each of them manually according to the testing                 One avenue for future work is investigating the long-term
+patterns they adhere to, which stem from an influential blog            impact of PBT-introducing test case evolutions. Another is
+post7 . They also study the use of Hypothesis features in               developer support in the form of a tool that identifies candidate
+those PBTs, similar to us. However, while they investigate              unit tests for such an evolution, and for partially automating
+bigger, established projects, we investigate the adoption of            the necessary test changes.
+these features in test cases that were previously not a PBT.
+                                                                                          ACKNOWLEDGEMENTS
+We also study whether those features are adopted later on,
+while the PBT matures.                                                    This research was partially funded by the Research Foun-
+                                                                        dation Flanders (FWO) Grant No. 1SHFI24N and by the
+  7 https://fsharpforfunandprofit.com/posts/property-based-testing-2/   Cybersecurity Research Program Flanders (CRPF).
+                              R EFERENCES                                        [18] John Hughes. Experiences with QuickCheck: Testing the Hard Stuff and
+                                                                                      Staying Sane, pages 169–186. Springer International Publishing, Cham,
+ [1] Maurı́cio Aniche, Christoph Treude, and Andy Zaidman. How develop-               2016.
+     ers engineer test cases: An observational study. IEEE Transactions on       [19] John Hughes, Benjamin C. Pierce, Thomas Arts, and Ulf Norell. Myster-
+     Software Engineering, 48(12):4925–4946, 2022.                                    ies of dropbox: Property-based testing of a distributed synchronization
+ [2] Thomas Arts, John Hughes, Joakim Johansson, and Ulf Wiger. Testing               service. In 2016 IEEE International Conference on Software Testing,
+     telecoms software with quviq quickcheck. In Proceedings of the 2006              Verification and Validation (ICST), pages 135–145, 2016.
+     ACM SIGPLAN Workshop on Erlang, ERLANG ’06, page 2–10, New                  [20] Laura Inozemtseva and Reid Holmes. Coverage is not strongly correlated
+     York, NY, USA, 2006. Association for Computing Machinery.                        with test suite effectiveness. In Proceedings of the 36th International
+ [3] Earl T. Barr, Mark Harman, Phil McMinn, Muzammil Shahbaz, and                    Conference on Software Engineering, ICSE 2014, page 435–445, New
+     Shin Yoo. The oracle problem in software testing: A survey. IEEE                 York, NY, USA, 2014. Association for Computing Machinery.
+     Transactions on Software Engineering, 41(5):507–525, 2015.                  [21] Paul Jansen. Tiobe index for december 2025. https://web.archive.org/
+                                                                                      web/20251218235222/https://www.tiobe.com/tiobe-index/. Accessed on
+ [4] Ned Batchelder and Contributors to Coverage.py. Coverage.py: The code
+                                                                                      20 december 2025.
+     coverage tool for Python. https://web.archive.org/web/20251223093400/
+                                                                                 [22] JetBrains.      Python developers survey 2023 results.             https:
+     https://github.com/coveragepy/coveragepy. Accessed on 20 december
+                                                                                      //web.archive.org/web/20250920050020/https://lp.jetbrains.com/
+     2025.
+                                                                                      python-developers-survey-2023/. Accessed on 10 december 2025.
+ [5] Laurent Christophe, Reinout Stevens, Coen De Roover, and Wolfgang
+                                                                                 [23] Holger Krekel, Bruno Oliveira, Ronny Pfannschmidt, Floris
+     De Meuter. Prevalence and maintenance of automated functional tests for
+                                                                                      Bruynooghe, Brianna Laugher, and Florian Bruhin.                  pytest
+     web applications. In 2014 IEEE International Conference on Software
+                                                                                      9.0.0.               https://web.archive.org/web/20250906051102/https:
+     Maintenance and Evolution, pages 141–150, 2014.
+                                                                                      //github.com/pytest-dev/pytest/, 2004.      Version 9.0.0 Contributors
+ [6] Koen Claessen and John Hughes. Quickcheck: a lightweight tool
+                                                                                      include Holger Krekel, Bruno Oliveira, Ronny Pfannschmidt, Floris
+     for random testing of haskell programs. In Proceedings of the Fifth
+                                                                                      Bruynooghe, Brianna Laugher, Florian Bruhin, and others.
+     ACM SIGPLAN International Conference on Functional Programming,
+                                                                                 [24] Leonidas Lampropoulos, Michael Hicks, and Benjamin C. Pierce. Cov-
+     ICFP ’00, page 268–279, New York, NY, USA, 2000. Association for
+                                                                                      erage guided, property based testing. Proc. ACM Program. Lang.,
+     Computing Machinery.
+                                                                                      3(OOPSLA), October 2019.
+ [7] Jacob Cohen. A coefficient of agreement for nominal scales. Educational     [25] J. Richard Landis and Gary G. Koch. The measurement of observer
+     and psychological measurement, 20(1):37–46, 1960.                                agreement for categorical data. Biometrics, 33(1):159–174, 1977.
+ [8] Arthur Lisboa Corgozinho, Marco Tulio Valente, and Henrique Rocha.          [26] David R. MacIver and Alastair F. Donaldson. Test-Case Reduction via
+     How Developers Implement Property-Based Tests . In 2023 IEEE Inter-              Test-Case Generation: Insights from the Hypothesis Reducer. In Robert
+     national Conference on Software Maintenance and Evolution (ICSME),               Hirschfeld and Tobias Pape, editors, 34th European Conference on
+     pages 380–384, Los Alamitos, CA, USA, October 2023. IEEE Computer                Object-Oriented Programming (ECOOP 2020), volume 166 of Leibniz
+     Society.                                                                         International Proceedings in Informatics (LIPIcs), pages 13:1–13:27,
+ [9] Ermira Daka and Gordon Fraser. A survey on unit testing practices and            Dagstuhl, Germany, 2020. Schloss Dagstuhl – Leibniz-Zentrum für
+     problems. In 2014 IEEE 25th International Symposium on Software                  Informatik.
+     Reliability Engineering, pages 201–211, 2014.                               [27] David R MacIver, Zac Hatfield-Dodds, et al. Hypothesis: A new
+[10] Albert Danial. cloc: v1.92. https://doi.org/10.5281/zenodo.5760077,              approach to property-based testing. Journal of Open Source Software,
+     December 2021.                                                                   4(43):1891, 2019.
+[11] Sergio Di Meglio, Luigi Libero Lucio Starace, Valeria Pontillo, Ruben       [28] Gerard Meszaros. xUnit test patterns: Refactoring test code. Pearson
+     Opdebeeck, Coen De Roover, and Sergio Di Martino. Performance                    Education, 2007.
+     testing in open-source web projects: Adoption, maintenance, and a           [29] Agustı́n Mista and Alejandro Russo. Mutagen: Reliable coverage-
+     change taxonomy. In Proceedings of the 41st IEEE International                   guided, property-based testing using exhaustive mutations. In 2023 IEEE
+     Conference on Software Maintenance and Evolution (ICSME 2025).                   Conference on Software Testing, Verification and Validation (ICST),
+     IEEE, sep 2025. 41st IEEE International Conference on Software                   pages 176–187, 2023.
+     Maintenance and Evolution (ICSME 2025), ICMSE ; Conference date:            [30] Rohan Padhye, Caroline Lemieux, and Koushik Sen. Jqf: coverage-
+     07-09-2025 Through 12-09-2025.                                                   guided property-based testing in java. In Proceedings of the 28th ACM
+[12] S. Elbaum, D. Gable, and G. Rothermel. The impact of software                    SIGSOFT International Symposium on Software Testing and Analysis,
+     evolution on code coverage information. In Proceedings IEEE Inter-               ISSTA 2019, page 398–401, New York, NY, USA, 2019. Association
+     national Conference on Software Maintenance. ICSM 2001, pages 170–               for Computing Machinery.
+     179, 2001.                                                                  [31] Hila Peleg, Dan Rasin, and Eran Yahav. Generating tests by example.
+[13] Khashayar Etemadi, Marjan Sirjani, Mahshid Helali Moghadam, Per                  In Isil Dillig and Jens Palsberg, editors, Verification, Model Checking,
+     Strandberg, and Paul Pettersson. Llm-based property-based test genera-           and Abstract Interpretation, pages 406–429, Cham, 2018. Springer
+     tion for guardrailing cyber-physical systems. In International Conference        International Publishing.
+     on Bridging the Gap between AI and Reality, pages 18–46. Springer           [32] Savitha Ravi and Michael Coblenz. An empirical evaluation of property-
+     Nature Switzerland Cham, 2025.                                                   based testing in python. Proc. ACM Program. Lang., 9(OOPSLA2),
+[14] Harrison Goldstein, Joseph W. Cutler, Daniel Dickstein, Benjamin C.              October 2025.
+     Pierce, and Andrew Head. Property-based testing in practice. In Pro-        [33] Mohammad Rezaalipour and Carlo A. Furia. An annotation-based
+     ceedings of the IEEE/ACM 46th International Conference on Software               approach for finding bugs in neural network programs. Journal of
+     Engineering, ICSE ’24, New York, NY, USA, 2024. Association for                  Systems and Software, 201:111669, 2023.
+     Computing Machinery.                                                        [34] Davide Spadini, Maurı́cio Aniche, and Alberto Bacchelli. Pydriller:
+[15] Harrison Goldstein, Joseph W Cutler, Adam Stein, Benjamin C Pierce,              Python framework for mining software repositories. In Proceedings of
+     and Andrew Head. Some problems with properties. In Proc. Workshop                the 2018 26th ACM Joint Meeting on European Software Engineering
+     on the Human Aspects of Types and Reasoning Assistants (HATRA),                  Conference and Symposium on the Foundations of Software Engineering,
+     2022.                                                                            ESEC/FSE 2018, page 908–911, New York, NY, USA, 2018. Associa-
+[16] Harrison Goldstein, Jeffrey Tao, Zac Hatfield-Dodds, Benjamin C.                 tion for Computing Machinery.
+     Pierce, and Andrew Head. Tyche: Making sense of pbt effectiveness. In       [35] Nikolai Tillmann and Wolfram Schulte. Parameterized unit tests.
+     Proceedings of the 37th Annual ACM Symposium on User Interface                   SIGSOFT Softw. Eng. Notes, 30(5):253–262, September 2005.
+     Software and Technology, UIST ’24, New York, NY, USA, 2024.                 [36] Nikolai Tillmann and Wolfram Schulte. Parameterized unit tests. In
+     Association for Computing Machinery.                                             Proceedings of the 10th European Software Engineering Conference
+[17] Michael Hilton, Jonathan Bell, and Darko Marinov. A large-scale study            Held Jointly with 13th ACM SIGSOFT International Symposium on
+     of test coverage evolution. In Proceedings of the 33rd ACM/IEEE                  Foundations of Software Engineering, ESEC/FSE-13, page 253–262,
+     International Conference on Automated Software Engineering, ASE ’18,             New York, NY, USA, 2005. Association for Computing Machinery.
+     page 53–63, New York, NY, USA, 2018. Association for Computing              [37] Deepika Tiwari, Yogya Gamage, Martin Monperrus, and Benoit Baudry.
+     Machinery.                                                                       Proze: Generating parameterized unit tests informed by runtime data.
+     In 2024 IEEE International Conference on Source Code Analysis and               tation. https://web.archive.org/web/20251223092313/https://hypothesis.
+     Manipulation (SCAM), pages 166–176, 2024.                                       readthedocs.io/en/latest/reference/api.html. Accessed on 10 december
+[38] Vasudev Vikram, Caroline Lemieux, Joshua Sunshine, and Rohan Pad-               2025.
+     hye. Can large language models write good property-based tests?, 2024.     [43] Hypothesis Works. When to use hypothesis and property-based testing
+[39] Cindy Wauters and Coen De Roover. Property-based Testing within ML              - hypothesis 6.148.3 documentation.          https://web.archive.org/web/
+     Projects: an Empirical Study . In 2024 IEEE International Conference            20251223092910/https://hypothesis.readthedocs.io/en/latest/tutorial/
+     on Software Maintenance and Evolution (ICSME), pages 648–653, Los               introduction.html#when-to-use-hypothesis-and-property-based-testing.
+     Alamitos, CA, USA, October 2024. IEEE Computer Society.                         Accessed on 11 december 2025.
+[40] Cindy Wauters, Ruben Opdebeeck, and Coen De Roover. Replication            [44] Andy Zaidman, Bart Van Rompaey, Serge Demeyer, and Arie van
+     package on the evolution of python test cases into property-based tests.        Deursen. Mining software repositories to study co-evolution of pro-
+     https://doi.org/10.6084/m9.figshare.31424447, Feb 2026.                         duction & test code. In 2008 1st International Conference on Software
+[41] Frank Wilcoxon. Individual comparisons by ranking methods. Biomet-              Testing, Verification, and Validation, pages 220–229, 2008.
+     rics Bulletin, 1(6):80–83, 1945.
+[42] Hypothesis Works. Api reference - hypothesis 6.148.3 documen-
+

packages/opencode/specs/simulation-research/text/2026-propgen-mobile-app-testing.txt

@@ -0,0 +1,859 @@
+                                                   From Exploration to Specification: LLM-Based Property
+                                                            Generation for Mobile App Testing
+                                                         Yiheng Xiong                                      Shiwen Song                                       Bo Ma
+                                              Singapore Management University                  Singapore Management University                  East China Normal University
+                                                         Singapore                                        Singapore                                        China
+                                                   yihengx98@gmail.com                               swsong@smu.edu.sg                             boma@stu.ecnu.edu.cn
+
+                                                                                      Ting Su                                    Xiaofei Xie
+                                                                         East China Normal University                Singapore Management University
+                                                                                    China                                       Singapore
+arXiv:2604.13463v1 [cs.SE] 15 Apr 2026
+
+
+
+
+                                                                              tsu@sei.ecnu.edu.cn                           xfxie@smu.edu.sg
+
+                                         Abstract                                                                    the functional correctness of mobile apps [24, 26]. However, it is
+                                         Mobile apps often suffer from functional bugs that do not cause             brittle, costly to maintain, and typically covers only pre-defined
+                                         crashes but instead manifest as incorrect behaviors under specific          happy paths, often missing non-trivial functional bugs [66].
+                                         user interactions. Such bugs are difficult to detect by conventional           Property-based testing (PBT) offers a promising direction for
+                                         automatic testing techniques because they often lack explicit test          addressing this challenge [5]. Recent work has demonstrated that
+                                         oracles. Property-based testing can effectively expose them by spec-        carefully designed properties can reveal non-trivial functional bugs
+                                         ifying intended behavior as properties and checking them under              that are missed by other techniques [52, 66]. In mobile app testing,
+                                         diverse interactions. However, its practical use is limited by the need     developers specify expected behaviors as properties, and a PBT
+                                         for manually written properties, which are difficult and expensive          framework then automatically generates a large number of GUI
+                                         to construct.                                                               events to explore diverse GUI states and check whether these prop-
+                                            To address this limitation, this paper explores the use of large         erties hold. Compared with validating only a fixed set of manually
+                                         language models (LLMs) to automate property construction for                crafted GUI test cases, this paradigm provides a more efficient way
+                                         property-based testing of mobile apps. This process is challenging          to assess functional correctness across diverse GUI states.
+                                         in two ways. First, it is difficult to systematically uncover and exe-         However, the effectiveness of PBT fundamentally depends on
+                                         cute diverse app functionalities. Second, it is difficult to derive valid   the availability of high-quality properties, whose manual construc-
+                                         properties from functionality execution results, because a single ex-       tion remains a major barrier to practical adoption [15, 16]. This
+                                         ecution provides only limited evidence about what behavior should           challenge is especially pronounced for mobile apps, where explicit
+                                         generally hold. To address these challenges, we introduce Prop-             behavioral specifications are often unavailable. As a result, testers
+                                         Gen, which performs functionality-guided exploration to collect             must manually understand app functionalities, abstract their ex-
+                                         behavioral evidence from execution results, synthesizes properties          pected behaviors into executable properties, and refine the proper-
+                                         from the collected evidence, and refines imprecise properties based         ties when reported violations are false positives. This manual and
+                                         on testing feedback. We implemented PropGen and evaluated it                iterative process substantially limits the broader adoption of PBT.
+                                         on 12 real-world Android apps. The results show that PropGen                   To address this limitation, a promising direction is to leverage
+                                         can effectively identify and execute app functionalities, generate          the reasoning and code-generation capabilities of Large Language
+                                         valid properties, and refine most imprecise ones. Across all apps,          Models (LLMs) to automate property construction. However, di-
+                                         PropGen identified 1,210 valid functionalities and correctly exe-           rectly asking an LLM to generate properties is often unreliable, due
+                                         cuted 977 of them, compared with 491 and 187 for the baseline. It           to both the lack of explicit behavioral specifications in mobile apps
+                                         generated 985 properties, 912 of which were valid, and successfully         and the tendency of LLMs to hallucinate. Instead, an effective solu-
+                                         refined 118 of 127 imprecise ones exposed during testing. Using the         tion should be able to identify app functionalities, infer properties
+                                         resulting properties, we found 25 previously unknown functional             from execution-derived behavioral evidence, and refine imprecise
+                                         bugs in these apps, many of which were missed by existing testing           properties based on testing feedback.
+                                         techniques.                                                                 Challenges. However, achieving this goal is far from straight-
+                                                                                                                     forward and presents two key challenges: broad functionality ex-
+                                         1    Introduction                                                           ploration and property abstraction from execution traces. First, the
+                                         Mobile apps are highly interactive and stateful systems whose func-         approach must explore as many meaningful app functionalities as
+                                         tionalities are largely driven by user interface interactions. Despite      possible, but this is difficult in mobile apps. Many functionalities are
+                                         extensive testing efforts, functional bugs (e.g., incorrect interaction     not directly visible on the current screen. They may only become
+                                         logic) remain prevalent in real-world apps [67]. These bugs often           available after several navigation steps, under specific UI states, or
+                                         do not manifest as crashes, making them difficult to detect using           through transient interface elements such as menus and dialogs. As
+                                         conventional GUI testing techniques that mainly emphasize code              a result, systematically exposing a large and diverse set of function-
+                                         coverage or crash discovery [8, 17, 29, 31, 33, 38, 47, 58, 61]. Manual     alities through app exploration is non-trivial. Without sufficient
+                                         testing (e.g., writing GUI tests) is widely used in practice to validate    functionality exposure, the generated properties can cover only
+Conference’17, July 2017, Washington, DC, USA                                                 Yiheng Xiong, Shiwen Song, Bo Ma, Ting Su, and Xiaofei Xie
+
+
+a limited portion of app behavior. Second, even after a function-             In summary, this paper makes the following contributions:
+ality is executed, it is still difficult to infer a valid property from
+                                                                          • We propose a novel approach that automatically explores app
+the execution traces. This is because a property must capture the
+                                                                            functionalities and generates properties from runtime behavioral
+essential behavior of the functionality, rather than merely describe
+                                                                            evidence, without relying on manual specifications.
+one observed execution. In practice, however, the execution trace
+                                                                          • We design a hypothesis-driven behavioral evidence construction
+(e.g., events and screenshots) is often noisy and low-level, and only
+                                                                            technique that infers functionalities from GUI states and sum-
+provides a single concrete behavioral instance. Moreover, the in-
+                                                                            marizes execution traces into structured representations suitable
+ferred property can become inaccurate: it may encode details that
+                                                                            for property synthesis.
+happen to hold in the current trace, but do not necessarily hold in
+                                                                          • We develop a property synthesis and refinement approach that
+other valid contexts. Such imprecision can lead to false positives
+                                                                            derives properties from behavior evidence and refines imprecise
+during testing and reduce the usefulness of the generated proper-
+                                                                            properties through execution feedback.
+ties. Importantly, this difficulty is also faced by human developers
+                                                                          • We implement our approach as PropGen and conduct an exten-
+when manually writing properties.
+                                                                            sive evaluation on 12 real-world Android apps. The results show
+Our approach. To address the first challenge, we design a function-
+                                                                            that PropGen can cover a large number of app functionalities
+ality hypothesis-guided exploration strategy that systematically
+                                                                            while achieving high functionality and property validity, and can
+uncovers executable app functionalities and collects behavioral evi-
+                                                                            effectively refine most imprecise properties.
+dence from their executions. Given a GUI state, our approach infers
+candidate functionalities and grounds each of them in concrete            2    Background
+actionable widgets, making the inferred functionality hypothesis
+                                                                          Property-based testing. Property-based testing is a powerful test-
+both reliable and directly executable. Based on these hypotheses,
+                                                                          ing methodology that validates whether a program satisfies general
+our approach performs targeted functionality execution. During
+                                                                          properties rather than specific input-output examples [5]. Instead
+this process, it incrementally expands the functionality pool when
+                                                                          of writing example-based test cases, testers specify high-level prop-
+new functionality appears, reuses previously inferred function-
+                                                                          erties that describe the expected behavior of the system. Then, a
+alities across recurrent GUI states, and falls back to lightweight
+                                                                          PBT framework automatically generates a large number of test
+random exploration only when no functionality is available. This
+                                                                          inputs and executes them to verify whether the properties hold.
+hybrid strategy improves functionality coverage and enables richer
+                                                                          For example, for a sorting function sort, rather than enumerating
+behavioral evidence collection under a limited exploration budget.
+                                                                          concrete examples (e.g., sort([3,1,2])== [1,2,3]), one can de-
+    To address the second challenge, PropGen adopts property gen-
+                                                                          fine a general property such as idempotence: sort(sort(x)) =
+eration from behavior evidence followed by feedback-driven re-
+                                                                          sort(x). The PBT framework then generates a large number of
+finement. Instead of generating properties directly from execution
+                                                                          inputs to verify this property and reports any violating input as a
+results, PropGen first abstracts each functionality execution trace
+                                                                          counterexample.
+into a compact condition–action–outcome representation, derives
+                                                                          GUI state, event, and functionality. Android applications are
+a structured property description, and then translates it into the
+                                                                          GUI-driven and event-based. When an app A runs, its runtime
+executable property. Because properties inferred from limited evi-
+                                                                          state is represented by its current GUI layout, which we denote as a
+dence may still over-generalize, PropGen further refines those that
+                                                                          GUI state 𝑠. A GUI state corresponds to a hierarchical tree ℓ, whose
+trigger false positives during testing. Each refinement is anchored
+                                                                          nodes are GUI widgets 𝑤 (e.g., Button, TextView, EditText) with
+to the behavioral evidence, allowing PropGen to localize whether
+                                                                          attributes (e.g., text, resourceId) and interaction capabilities.
+the issue lies in the precondition, interaction, or postcondition
+                                                                             User interactions are modeled as events. An event is defined as
+and apply a targeted refinement. Through this process, PropGen
+                                                                          𝑒 = ⟨𝑡, 𝑤, 𝑑⟩, where 𝑡 is the event type (e.g., click), 𝑤 is the target
+improves property precision and robustness while preserving the
+                                                                          widget, and 𝑑 is optional data (e.g., text input). An app execution is
+original intent.
+                                                                          modeled as a sequence of events. Given 𝐸 = [𝑒 1, . . . , 𝑒𝑛 ], executing
+Evaluation and results. We implemented our approach as a tool                                            𝑒1              𝑒𝑛
+                                                                          A produces a trace 𝜏 = 𝑠 0 −→ 𝑠 1 →   − · · · −−→ 𝑠𝑛 , or 𝑠 0 ⇝ 𝑠𝑛 , where
+                                                                                                                                         𝐸
+named PropGen and evaluated it on 12 real-world popular and
+diverse Android apps. Across all apps, PropGen inferred 1,282 func-       𝑠 0 is the initial state. Then, we define a functionality as a tuple 𝑓 =
+tionalities, of which 1,210 (94.4%) were valid and 977 (76.2%) were       ⟨𝑑, 𝜏⟩, where 𝑑 is a semantic intent of a functionality (e.g., "create a
+correctly executed. In comparison, the baseline approach inferred         note"), and 𝜏 is an execution trace that realizes this functionality.
+575 functionalities, with 491 valid and 187 correctly executed. It fur-   Concretely, 𝜏 has the form 𝑠 ⇝ 𝑠 ′ , where 𝐸 is a sequence of one or
+                                                                                                          𝐸
+
+ther generated 985 property descriptions, 912 (92.6%) of which were       more events executable from GUI state 𝑠. Executing 𝜏 completes
+valid. During property-based testing, 127 properties were found           the corresponding functionality and produces a concrete effect on
+to be imprecise, and 118 (93.7%) of them were successfully refined        the app state.
+by our refinement technique. Using the generated properties, we           Property-based testing for mobile apps. In property-based test-
+found 25 previously unknown functional bugs in the latest versions        ing for mobile apps, a property specifies an expected behavior of
+of the subject apps, whereas existing functional testing techniques       the app. A property can be defined as a tuple 𝜙 = ⟨𝑃, 𝐼, 𝑄⟩, where 𝑃
+could find only 3 of them in practice. These results demonstrate          specifies the GUI states where the property applies, 𝐼 is the interac-
+the effectiveness of our approach in automating property construc-        tion scenario, and 𝑄 specifies the expected outcome after executing
+tion for mobile apps, as well as the bug-finding capability of the        𝐼 . During testing, when a GUI state 𝑠 satisfies 𝑃, the interaction
+resulting properties.                                                     scenario 𝐼 is executed from 𝑠 to reach a new state 𝑠 ′ . The property
+From Exploration to Specification: LLM-Based Property Generation for Mobile App Testing                                 Conference’17, July 2017, Washington, DC, USA
+
+
+is satisfied when (𝑠 |= 𝑃 ∧𝑠 ⇝ 𝑠 ′ ) ⇒ 𝑠 ′ |= 𝑄; Otherwise, a property
+                                 𝐼
+                                                                                          Algorithm 1 Behavioral Evidence Construction
+violation is reported.                                                                    Require: Target app A, time budget 𝐵
+                                                                                          Ensure: Behavioral evidence set T̂
+                                                                                           1: Launch A; obtain initial state 𝑠
+3     Approach
+                                                                                           2: G𝐹 ← ∅, U ← ∅, T̂ ← ∅
+Overview. Given an app, PropGen automatically generates prop-                              3: while elapsed time < 𝐵 do
+erties from runtime executions and further refines imprecise ones                          4:    𝑢 (𝑠) ← Extract(𝑠)                         ⊲ Extract the widgets
+based on testing feedback. Figure 1 presents the overall workflow,                         5:    if UnseenWidgets(𝑢 (𝑠), U) then
+which consists of three stages. First (§3.1), PropGen performs                             6:         𝐹 (𝑠) ← Inferhypothesis(𝑠)           ⊲ infer the hypothesis
+hypothesis-driven dynamic exploration to construct behavioral                              7:         G𝐹 ← G𝐹 ∪ 𝐹 (𝑠) ⊲ add the hypothesis into global pool
+evidence. For each encountered GUI state 𝑠, it infers functionality                        8:         U ← U ∪ 𝑢 (𝑠)
+hypothesis grounded in the visible UI widgets, executes selected                           9:    end if
+functionalities, and summarizes the resulting execution traces into                       10:    if HasUnexplored(G𝐹 (𝑢 (𝑠))) then
+behavioral evidence for downstream property generation. Second                            11:         𝑓 ← Select(G𝐹 (𝑢 (𝑠)))
+(§3.2), PropGen synthesizes properties directly from the behavioral                       12:         (𝜏 𝑓 , 𝑠 ′, G𝐹 , U) ← Execute(𝑓 , 𝑠, G𝐹 , U)
+evidence constructed in the first stage. Specifically, it first generates                 13:         T̂ ← T̂ ∪ {Summarize(𝜏 𝑓 )}            ⊲ evidence summary
+natural-language property descriptions that capture the intended                          14:         MarkExplored(𝑓 , G𝐹 )
+condition–event–outcome relation, and then translates them into                           15:        𝑠 ← 𝑠′
+executable properties. Third (§3.3), PropGen validates the gener-                         16:    else
+ated properties by the PBT framework and refines properties that                          17:        𝑠 ← RandomExplore(𝑠)
+are found to be improperly specified.                                                     18:    end if
+Illustrative example. Figure 2 illustrates the workflow of Prop-                          19: end while
+Gen on a note-taking app. Starting from the first page in Figure 2                        20: return T̂
+(a), PropGen performs behavioral evidence construction by infer-
+ring multiple candidate functionality hypotheses from the current
+GUI state. It then selects one hypothesis, attaching a photo to the
+note, and executes the corresponding interaction sequence, i.e.,                              Given a target app A and a time budget 𝐵, PropGen first launches
+opening the attachment menu, choosing Camera, taking a photo,                             the app and initializes the global functionality pool G𝐹 , the set of
+and returning to the note page. The execution trace is summarized                         observed UI contexts U, and the behavioral evidence set T̂ (Lines 1–
+into structured behavioral evidence (as shown in Figure 2(c)), from                       2). It then iteratively explores the app until the budget is exhausted
+which PropGen synthesizes an executable property describing the                           (Lines 3–19). In each iteration, PropGen extracts the current GUI
+expected behavior of this functionality (Figure 2(d)).                                    context from the current GUI state (Line 4). If the context contains
+   The initial synthesized property may be imprecise and thus                             previously unseen widget evidence, PropGen invokes a Multimodal
+produce false positives during execution. In this example, its post-                      Large Language Model (MLLM) to infer functionality hypotheses
+condition checks whether the app returns to a page containing the                         for the current state, adds them to G𝐹 , and updates U (Lines 5–8).
+text “Notes” and whether the newly added attachment is displayed                          PropGen then checks whether the current context contains any
+as an attachment thumbnail. This postcondition is too specific. Af-                       unexplored functionality hypothesis (Line 10). If so, it selects one
+ter taking a photo, the app may legitimately return to either the                         hypothesis together with its triggering widget, executes it to obtain
+Notes page or the Archive page, depending on where the note was                           a functionality trace, and summarizes the trace into a behavioral
+opened. Moreover, under reduced view, the attached photo may not                          evidence item added to T̂ (Lines 11–15). Otherwise, PropGen per-
+appear as a thumbnail, but instead as a compact attachment icon                           forms lightweight random exploration to leave the current local
+(Figure 2(b)). Therefore, the synthesized property may incorrectly                        GUI region and expose new interaction opportunities (Lines 16–
+flag a failure even though the photo has been successfully attached.                      17). The process repeats until the time budget is exhausted, after
+PropGen then refines the property by relaxing these assertions                            which PropGen returns the collected behavioral evidence set T̂
+to allow multiple valid return pages (use the menu button that                            (Lines 19–20).
+both pages contain) and attachment representations (thumbnail or
+icon). After validating this property using the PBT tool Kea, we                          3.1.1 Functionality Hypothesis Generation. Given a GUI state 𝑠
+uncovered a new functional bug: opening audio recording before                            encountered during exploration, this step aims to infer which app
+taking a photo prevents the photo attachment from appearing.                              functionalities may be executable under the current interface con-
+                                                                                          text. To support subsequent execution, each inferred functionality
+                                                                                          is associated with a concrete triggering widget in the current state.
+3.1     Behavioral Evidence Construction                                                  Semantic context construction. To support functionality infer-
+The goal of this stage is to construct execution-grounded behavioral                      ence, PropGen first constructs a semantic context for the current
+evidence for downstream property synthesis. Instead of relying on                         GUI state 𝑠. This context includes three parts: the screen-level con-
+external specifications, PropGen systematically explores executable                       text of the current state, app-level semantic cues, and cross-state
+app functionalities and records their runtime interaction traces as                       functionality memory. The screen-level context is derived from
+behavioral evidence. Algorithm 1 summarizes the workflow.                                 the current GUI screenshot, where each interactive widget 𝑤 in 𝑠
+Conference’17, July 2017, Washington, DC, USA                                                                    Yiheng Xiong, Shiwen Song, Bo Ma, Ting Su, and Xiaofei Xie
+
+
+
+                       Behavioral Evidence Construction                                              Property Synthesis from Behaviroal Evidence
+
+                                                                                          Property Description Generation                Executable Property Translation
+                         Semantic Context
+                                                    Functionality selection
+                           Construction
+
+                                                                                                 Feedback-Driven Property Refinement                          Output
+                                                      Event Planing and
+                       Hypothesis Inference
+                                                         Execution                         Executable Property              Validation by PBT tool            Property
+                                                                                                                                                             Description
+                                                                                                                       Failed Infomation
+                                                             Behavioral                                                Imprecise Property
+                                                                                                Property
+                           Functionality                     Evidence
+  App Under Test                                                                               Refinement
+                            Hypothesis                     Summarization                                                                                     Executable
+                                                                                                                        Failure Evidence                      Property
+
+
+
+                                                              Figure 1: Overview of PropGen.
+
+
+
+
+                                                (a) An execution trace of the functionality.                                                    (b) Suspicious violation.
+  ① Behavior Evidence Construction
+
+
+
+                                                                          ② Property
+                                                                              Synthesis
+                                                                                                                             ③ Feedback-Driven Property Refinement
+
+                       (c) Behavior evidence.                                                                     (d) Generated property.
+
+
+Figure 2: An Illustrative Example of Behavioral Evidence Construction, Property Synthesis, and Feedback-Driven Property
+Refinement for a Note-Taking App.
+
+
+is annotated with a unique numeric label. These labels turn visu-                         ⟨𝑓𝑘 , 𝑤𝑘 ⟩], where each functionality hypothesis 𝑓𝑖 is a concise natural-
+ally distributed UI elements into explicit references, allowing the                       language description of a candidate app functionality, and 𝑤𝑖 de-
+inferred functionalities to be grounded in concrete widgets. The                          notes its corresponding triggering widget. This widget grounding
+app-level semantic cues include the app name and the list of activity                     serves two purposes. First, it constrains the MLLM to infer func-
+names extracted from the AndroidManifest.xml file, helping the                            tionalities that are supported by the current GUI state, reducing
+MLLM interpret the current screen under the broader semantic                              unsupported or non-actionable predictions. Second, it makes the in-
+context of A. Finally, the cross-state functionality memory stores                        ferred functionalities directly executable in subsequent exploration.
+previously inferred functionalities, helping avoid repeatedly redis-                      The inferred functionality hypothesis, together with their associ-
+covering semantically similar functionalities in later GUI states.                        ated triggering widgets, are then stored in the global functionality
+Together, these components provide the contextual information                             pool G𝐹 to support later execution and cross-state reuse.
+needed for inferring plausible user-facing functionalities from 𝑠.                        Inference triggering and hypothesis reuse. To avoid redun-
+Functionality hypothesis inference. Using the constructed se-                             dant MLLM invocations on similar GUI states, PropGen triggers
+mantic context, PropGen invokes a MLLM to infer candidate func-                           functionality inference only when a newly visited state introduces
+tionality hypothesis for the current GUI state 𝑠. Formally, the in-                       unseen widget evidence. It maintains a global set of explored UI
+ferred hypothesis are represented as 𝐹 (𝑠) = [⟨𝑓1, 𝑤 1 ⟩, ⟨𝑓2, 𝑤 2 ⟩, . . . ,             contexts U = {𝑢 1, 𝑢 2, . . .}, where each context is defined as 𝑢 (𝑠) =
+From Exploration to Specification: LLM-Based Property Generation for Mobile App Testing                                 Conference’17, July 2017, Washington, DC, USA
+
+
+⟨𝑎(𝑠),𝑊 (𝑠)⟩, with 𝑎(𝑠) denoting the activity and 𝑊 (𝑠) the set of                        widget 𝑤𝑖 is specified by its numeric label, and 𝑑𝑖 ) specifies the
+signatures of leaf-level interactive widgets.                                             attached data (e.g., input text). Before executing 𝑒𝑖 , PropGen ver-
+   For each state 𝑠, PropGen compares 𝑊 (𝑠) with existing con-                            ifies that the referenced widget 𝑤𝑖 is present in the current state
+texts under the same activity. If no new widget signatures are                            𝑠𝑖 . If the predicted widget identifier is invalid, the event is skipped
+observed, it reuses previously inferred functionality hypotheses;                         and directly labeled as a failed step. This guarded execution mecha-
+otherwise, it invokes the MLLM to infer new functionalities and                           nism prevents invalid model-predicted interactions and improves
+updates the global pool G𝐹 . To enable stable comparison across                           execution robustness.
+                                                                                                                                                 𝑒𝑖
+screens, each widget 𝑤 is represented by a signature of attributes                             After execution produces a state transition 𝑠𝑖 −→ 𝑠𝑖+1 , PropGen
+⟨𝑐𝑙𝑎𝑠𝑠, 𝑟𝑒𝑠𝑜𝑢𝑟𝑐𝑒𝐼𝑑, 𝑡𝑒𝑥𝑡, 𝑑𝑒𝑠𝑐𝑟𝑖𝑝𝑡𝑖𝑜𝑛⟩, which are commonly used to                        evaluates whether the step has advanced the selected functionality.
+identify unique widgets in practice [47, 49, 64]. To reduce noise                         This evaluation jointly considers the pre- and post-event GUI states,
+from dynamic content, PropGen retains only app-defined text in                            the functionality hypothesis 𝑓 , and the accumulated execution his-
+signatures by filtering widget text against a whitelist extracted via                     tory 𝐻𝑖 . Based on this evidence, the MLLM assigns an outcome
+static analysis. This avoids treating semantically identical screens                      label 𝑜𝑖 ∈ {success, fail, complete} to the current step. Here, success
+with transient text differences as distinct contexts.                                     indicates meaningful progress toward the functionality goal, fail
+                                                                                          indicates that the event was unproductive for the intended func-
+3.1.2 Hypothesis-Guided Functionality Execution. Given the global                         tionality, and complete indicates that the functionality goal has been
+functionality pool G𝐹 generated in § 3.1.1, the goal of this step is                      achieved. The resulting step and outcome are then incorporated
+to expand behavioral coverage under a limited exploration budget                          into the execution history for subsequent planning, while complete
+by executing app functionalities in a targeted manner. Instead of                         terminates execution of the current functionality.
+interacting with A through arbitrary GUI events, PropGen treats                           Behavioral evidence summarization. To support downstream
+each functionality hypothesis in G𝐹 , together with its associated                        property synthesis, PropGen summarizes each executed interac-
+triggering widget, as an explicit execution target, and prioritizes                       tion trace into compact behavioral evidence. This step is necessary
+the execution of previously unexplored functionality hypothesis.                          because raw execution traces are low-level, noisy, and specific to a
+   Executing a selected functionality hypothesis ⟨𝑓 , 𝑤⟩ may require                      single execution, whereas property synthesis requires a higher-level
+one or more concrete GUI events, thereby inducing state transitions                       representation of the condition–event–outcome relation exhibited
+                𝐸
+of the form 𝑠 → − 𝑠 ′ . Such functionality-guided execution serves two                    by the app behavior. To bridge this gap, PropGen incrementally
+purposes simultaneously: it exercises already identified app behav-                                                           𝑒𝑖
+                                                                                          converts each interaction step 𝑠𝑖 −→ 𝑠𝑖+1 into a structured transition
+iors to collect behavioral evidence, and it drives the app into new                       with five elements: the state summary before the event, an event
+GUI states 𝑠 ′ from which additional functionality hypothesis 𝐹 (𝑠 ′ )                    summary describing the interaction performed, the state summary
+can be inferred and inserted into G𝐹 . In this way, execution and                         after the event, a state-diff summary capturing the visible difference
+hypothesis generation form a closed exploration loop that progres-                        between the two GUI states, and an outcome label 𝑜𝑖 indicating
+sively broadens the functionality space explored by PropGen.                              how the event affected progress toward the selected functionality.
+Functionality selection. At each GUI state 𝑠, PropGen retrieves                                The state summaries are functionality-oriented: they describe the
+from G𝐹 the candidate functionality hypothesis associated with                            screen context, visible actionable elements, current content state,
+the current UI context 𝑢 (𝑠), denoted as G𝐹 (𝑢 (𝑠)). Since the explo-                     and observable feedback cues. In contrast, the state-diff summary
+ration budget is limited, PropGen does not execute these candi-                           focuses on the concrete GUI changes induced by the interaction.
+dates arbitrarily, but prioritizes those with higher expected explo-                      Together, these summaries preserve the behavioral evidence needed
+ration utility. This prioritization is implemented using lightweight                      to capture the executed functionality’s condition–event–outcome
+heuristics guided by three considerations: whether the functional-                        relation while filtering out incidental GUI details. The resulting
+ity corresponds to a main app behavior, whether it is semantically                        summarized trace, denoted as 𝜏ˆ𝑓 , serves as the behavioral evidence
+different from previously executed functionalities, and whether it                        used in the subsequent property synthesis stage.
+is likely to be successfully executed in the current context. Based                       Exploration beyond local hypothesis exhaustion. As explo-
+on these heuristics, the top-ranked unexplored functionality hy-                          ration proceeds, PropGen may reach GUI states where all available
+pothesis ⟨𝑓 , 𝑤⟩ is selected as the next execution target. The selected                   functionality hypothesis have already been executed and no new
+hypothesis is then passed to a goal-directed interaction loop that                        hypothesis can be triggered. In such cases, continued MLLM-guided
+incrementally plans, executes, and evaluates UI actions until the                         exploration becomes less effective, because the MLLM is most use-
+functionality is completed or a step limit is reached.                                    ful when acting toward an explicit functionality goal. When no
+Event planning and execution. Once a functionality hypothesis                             such goal is available, the task is no longer to reason about how to
+⟨𝑓 , 𝑤⟩ is selected, PropGen executes it through a goal-directed in-                      execute a functionality, but simply to move the app into a new GUI
+teraction loop consisting of event planning, guarded execution, and                       region where new functionality hypothesis may emerge. For this
+post-event evaluation. At each step 𝑖, PropGen first predicts the                         purpose, lightweight random exploration is both more efficient and
+next GUI event based on three sources of information: the function-                       less costly. Therefore, once local functionality-guided exploration is
+ality goal 𝑓 , the execution history 𝐻𝑖 of prior events and their out-                    exhausted, PropGen switches to random exploration. It continues
+comes, and the current GUI state 𝑠𝑖 , represented as a screenshot with                    traversing the app through random GUI events until it reaches a
+labeled widgets. The predicted event is denoted as 𝑒𝑖 = ⟨𝑡𝑖 , 𝑤𝑖 , 𝑑𝑖 ⟩,                  state that either contains unexecuted functionality hypothesis or
+where the event type 𝑡𝑖 is chosen from a predefined event space                           exposes unseen widget evidence for new hypothesis generation.
+including click, long-click, edit, swipe, and back, the target
+Conference’17, July 2017, Washington, DC, USA                                                  Yiheng Xiong, Shiwen Song, Bo Ma, Ting Su, and Xiaofei Xie
+
+
+In the former case, PropGen resumes functionality-guided execu-            already been made explicit in the preceding natural-language for-
+tion directly; in the latter, it first performs functionality hypothesis   mulation step, this translation primarily serves to operationalize the
+generation and then continues execution. This design combines              structured specification rather than to perform further behavioral
+the strength of goal-directed MLLM-guided execution with the effi-         inference.
+ciency of random exploration for escaping locally exhausted GUI               This step is implemented by prompting the LLM with the gen-
+regions.                                                                   erated natural-language specification together with framework-
+                                                                           specific APIs and widget attributes, with reference to the prompt
+3.2     Property Synthesis from Behavior Evidence                          design in prior work on translating natural-language properties
+The goal of this stage is to derive executable properties from the         into executable ones [65]. The resulting code is then passed to the
+behavioral evidence collected during functionality exploration. As         subsequent validation and refinement stage.
+input, PropGen takes the summarized trace 𝜏ˆ𝑓 produced in Sec-
+tion 3.1, which compactly captures how a functionality is exercised
+and what observable GUI outcome it induces. Each synthesized
+                                                                           3.3    Feedback-Driven Property Refinement
+property takes the form 𝜙 = ⟨𝑃, 𝐼, 𝑄⟩, where 𝑃 is a precondition, 𝐼        Properties automatically generated from functionality traces may
+is an interaction scenario, and 𝑄 is a postcondition assertion.            still be imprecise and therefore trigger false positives during test-
+   Rather than generating 𝜙 directly from 𝜏ˆ𝑓 , PropGen decomposes         ing. However, a reported failure should not be refined by simply
+property synthesis into two steps, separating semantic property            adapting the property to that single execution, because such a fix
+formulation from executable code generation. It first constructs a         may overfit the observed case and drift away from the original
+natural-language property specification that explicitly captures the       functionality intent.
+intended ⟨𝑃, 𝐼, 𝑄⟩ relation, and then translates this intermediate            Our key idea is that refining a false-positive-inducing property
+specification into executable property code.                               requires first recovering the property’s original testing intent, and
+Natural-language property description generation. Based on                 then revising it so that it remains valid for both the original intended
+the summarized trace 𝜏ˆ𝑓 , PropGen first prompts the MLLM to con-          execution and the newly observed legitimate execution. To this end,
+struct a natural-language property specification 𝜙 𝑁 𝐿 = ⟨𝑃, 𝐼, 𝑄⟩ for     we ground refinement in the source summarized trace from which
+the functionality captured by the behavioral evidence. The sum-            the property was originally inferred, rather than treating refinement
+marized trace provides structured evidence about the execution             as unconstrained rewriting.
+context, performed interactions, observable state changes, and step           Specifically, PropGen reasons about the refinement using both
+outcomes. Rather than merely restating these observations, the             the source summarized trace and the failure-triggering execution.
+MLLM is guided to abstract from 𝜏ˆ𝑓 a generalized behavioral rule          Specifically, it first recovers the original testing intent of the prop-
+that should hold across executions of the same functionality.              erty by locating the relevant segment in 𝜏ˆ𝑓 that matches the prop-
+   The inferred specification 𝜙 𝑁 𝐿 consists of three parts: a pre-        erty’s triggering context, interaction scenario, and expected out-
+condition 𝑃 describing the observable GUI context in which the             come. It then compares this trace-grounded intended behavior with
+property should be checked, an interaction scenario 𝐼 capturing the        the failing execution to explain why the violation occurs and iden-
+user events needed to exercise the functionality, and a postcondition      tify which component of 𝜙 has become imprecise, i.e., the precon-
+𝑄 specifying the immediate visible effect that should hold after-          dition 𝑃, the interaction scenario 𝐼 , or the postcondition 𝑄.
+ward. To improve precision and executability, PropGen constrains              Based on this diagnosis, PropGen refines only the faulty compo-
+this inference process in three ways. First, 𝑃 must be grounded in         nent through a minimal modification. More specifically, it strength-
+observable UI evidence and sufficiently specific to avoid triggering       ens 𝑃 when additional UI guards are needed, revises 𝐼 when the
+the property on unrelated screens with superficially similar wid-          event sequence does not faithfully reflect the intended behavior, and
+gets. Second, 𝑄 must focus on effects that are directly and reliably       relaxes or simplifies 𝑄 when the assertion is overly specific. In this
+verifiable from the GUI, such as the appearance, disappearance, or         way, the refined property remains consistent with both the source-
+modification of visible widgets or content. Third, the inferred speci-     trace execution and the newly observed legitimate execution, while
+fication should avoid trace-specific brittle details, such as incidental   preserving the original testing intent as much as possible.
+text instances or unstable widget states, and instead capture func-
+tionality semantics in a form that remains robust across executions.
+   In this way, PropGen lifts concrete execution evidence into a           4     Implementation
+generalized, testable property abstraction. The resulting natural-         We implemented PropGen as an end-to-end prototype for auto-
+language specification makes the intended property semantics ex-           mated property generation on Android apps. The system is pri-
+plicit before code generation, thereby separating behavioral under-        marily written in Python and JavaScript, and integrates three key
+standing from executable realization.                                      capabilities: GUI state acquisition and interaction, multimodal LLM-
+Executable property translation. Given the inferred natural-               based reasoning, and executable property generation for Kea [66].
+language property specification 𝜙 𝑁 𝐿 = ⟨𝑃, 𝐼, 𝑄⟩, PropGen trans-          At runtime, PropGen uses uiautomator2 [56] to retrieve GUI layouts
+lates it into executable property code for the target PBT frame-           and Android Debug Bridge (ADB) [53] to capture screenshots and
+work. Specifically, this step realizes the precondition 𝑃, interaction     issue GUI actions, including click, long-click, edit, swipe, and
+scenario 𝐼 , and postcondition 𝑄 using the framework’s property            back. The generated properties are translated into the executable
+structure and API conventions, thereby producing a runnable prop-          format expected by Kea and can be directly executed within its
+erty implementation. Since the intended property semantics have            property-based testing workflow.
+From Exploration to Specification: LLM-Based Property Generation for Mobile App Testing                                 Conference’17, July 2017, Washington, DC, USA
+
+
+Table 1: App subjects used in our experiment (K=1,000,                                    hours for behavioral evidence construction and property synthesis.
+M=1,000,000)                                                                              After that, we used Kea to perform property-based testing with
+                                                                                          the generated properties for 6 hours per app, following the testing
+    App Name        App Feature            #Downloads       #Stars       LOC              budget adopted in Kea’s paper [66].
+    OmniNotes       Note Manager             100∼500K         2.8K      57,529            Baselines. We use five baselines for different research questions,
+    Markor          Text Editor                      -        5.3K      79,749
+    RetroMusic      Audio Player                 1∼5M           5K     110,065            according to their evaluation goals. For RQ1, we compare Prop-
+    Amaze           File Manager                 1∼5M         6.1K     159,040            Gen with DroidAgent [69], a representative LLM-based mobile
+    MyExpenses      Financial Assistant          1∼5M         1.1K     317,899            app functionality exploration approach, which can automatically
+    AntennaPod      Podcast Manager              1∼5M         7.7K     130,925
+    AnkiDroid       Flashcards Manager         10∼50M        10.9K     403,785
+                                                                                          identify and execute functionalities in mobile apps. For RQ4, which
+    OuterTune       Youtube Music Player             -        4.8K      89,673            evaluates bug-finding capability, we compare PropGen with four
+    NewPipe         Video Player                     -       37.5K     187,187            representative prior techniques for Android functional bug detec-
+                    Storage Browser              1∼5M           8K      95,705
+    MaterialFiles
+                                                                                          tion: Genie [48], Odin [60], PBFDroid [49], and VisionDroid [30].
+    Orgzly          To-do Lists Manager      100∼500K         2.8K      72,042
+    uhabits         Habit Tracker               5∼10M         9.7K      69,652            Among them, Genie and Odin rely on designed automated oracles,
+                                                                                          PBFDroid is a property-based testing technique for data manipu-
+                                                                                          lation functionalities (DMFs), and VisionDroid is an LLM-based
+5      Evaluation                                                                         multi-agent approach for functional bug detection.
+We evaluate PropGen from four perspectives: functionality explo-                          Evaluation method of RQ1. RQ1 aims to evaluate whether Prop-
+ration and property generation, property refinement, bug detection,                       Gen can correctly infer and execute app functionalities and synthe-
+and comparison with prior related techniques. Accordingly, we                             size valid properties from the collected behavioral evidence. This
+investigate whether PropGen can accurately explore app func-                              evaluation cannot be performed fully automatically, because mo-
+tionalities and generate semantically correct executable properties,                      bile apps typically lack precise functional specifications. Therefore,
+to what extent the generated properties suffer from imprecision                           we manually assess the validity of both the inferred functionali-
+that leads to false positives and whether such imprecision can be                         ties and the generated property descriptions, following prior work
+refined, whether the resulting properties can help uncover new                            on functionality-level evaluation beyond structural coverage met-
+functional bugs in real-world apps, and how PropGen compares                              rics [6, 69]. The manual evaluation mainly involves two annotators,
+with existing functional testing techniques in uncovering such bugs.                      who are graduate students majoring in software engineering and
+We formulate the following research questions:                                            with at least four years of Android app development experience. Be-
+                                                                                          fore the annotation, each annotator was given time (at least fifteen
+• RQ1: How effective is PropGen in exploring app functionalities
+                                                                                          minutes) to familiarize themselves with the overall functionalities
+  and generating valid properties?
+                                                                                          of every subject app. During this process, annotators also referred
+• RQ2: To what extent do generated properties suffer from im-
+                                                                                          to the app’s official introduction page when available, so that the
+  precision that leads to false positives, and how effective is our
+                                                                                          subsequent judgments were made with sufficient understanding of
+  refinement technique in refining them?
+                                                                                          the app’s functionalities. In addition, during annotation, annotators
+• RQ3: Can the generated properties help find new functional bugs
+                                                                                          could interact with the running app at any time to verify uncertain
+  in real-world mobile apps?
+                                                                                          cases.
+• RQ4: How does PropGen compare with prior functional testing
+                                                                                             For each inferred functionality, we evaluate two aspects. The
+  techniques in uncovering new functional bugs?
+                                                                                          first is functionality validity, which examines whether the inferred
+                                                                                          functionality actually exists in the app. Annotators are given the
+5.1       Setup and Method                                                                inferred functionality description together with the GUI screenshot
+App subjects. We selected 12 popular and representative open-                             from which it was inferred, and determine whether the described
+source Android apps as experimental subjects. Among them, eight                           functionality is genuinely supported by the interface. The second
+apps were adopted from prior studies on functional bug detection                          is execution correctness, which examines whether the system cor-
+for Android apps [48, 52, 60, 66]. From the candidate apps used                           rectly executes the inferred functionality. For each inferred func-
+in these studies, we excluded those that were either (1) no longer                        tionality, annotators are provided with the corresponding execution
+runnable or actively maintained, or (2) highly similar in functional-                     screenshots and interaction events, and judge whether the executed
+ity to apps already selected. To further improve subject diversity, we                    interaction sequence indeed realizes the intended functionality.
+additionally included four popular open-source apps from Google                              For each generated property description, we assess its valid-
+Play that provide different features. Table 1 summarizes the selected                     ity from three aspects: (1) whether the precondition appropriately
+apps. In the table, App Feature denotes the primary functionality                         constrains the UI state in which the property should be applied,
+of each app, #Downloads and #Stars report the number of Google                            (2) whether the interaction scenario accurately reflects the user in-
+Play installations and GitHub stars, respectively, and LOC gives the                      teractions required to perform the functionality, and (3) whether
+lines of code.                                                                            the postcondition correctly captures the observable UI behavior
+Experimental environment. All experiments were conducted on                               that should hold immediately after the scenario. At the same time,
+a machine running Ubuntu 22.04 with 192 CPU cores (AMD EPYC                               because each property is abstracted from an original functional-
+9654) and official Android emulators (Android 11, Pixel). We use                          ity execution trace, it should first hold on the source trace from
+GPT-5.2 as the backend MLLM with default settings for PropGen                             which it is derived. Based on this principle, a property description
+and baseline tools that involve LLM. For each app, we allocated 3                         is considered valid if it faithfully reflects the behavior exhibited in
+Conference’17, July 2017, Washington, DC, USA                                                         Yiheng Xiong, Shiwen Song, Bo Ma, Ting Su, and Xiaofei Xie
+
+
+the source trace and can serve as a reasonable specification of the     Table 2: Validity of behavioral evidence and synthesized prop-
+intended functionality; otherwise, it is labeled as invalid.            erties across subject apps (RQ1).
+   Two annotators independently performed the annotations. We
+measured inter-rater agreement using Cohen’s 𝜅, obtaining 0.91,          App Name
+                                                                                        #Inferred Func           #Valid Func           #Correctly Executed
+                                                                                                                                                                  #Prop
+                                                                                                                                                                            #Valid
+
+0.82, and 0.81 for functionality validity, execution correctness, and                   D        P           D                 P          D             P                    Prop
+
+                                                                         OmniNotes      53      108      53 (100.0%)    102 (94.4%)   18 (34.0%)    80 (74.1%)     81     80 (98.8%)
+property validity, respectively. Disagreements were resolved through      Markor        48      117      36 (75.0%)     116 (99.1%)   12 (33.3%)    85 (72.6%)     88     83 (94.3%)
+discussion with authors. Based on these annotations, we report           RetroMusic     48      113      42 (87.5%)     104 (92.0%)    9 (21.4%)    88 (77.9%)     89     70 (78.7%)
+
+the proportions of inferred functionalities that are valid, inferred       Amaze        37      84       24 (64.9%)     83 (98.8%)     8 (33.3%)    64 (76.2%)     57     52 (91.2%)
+                                                                                        54      106      54 (100.0%)    104 (98.1%)   21 (38.9%)    82 (77.4%)     81     78 (96.3%)
+functionalities that are correctly executed, and generated property
+                                                                         MyExpenses
+                                                                         AntennaPod     47      142      38 (80.9%)     139 (97.9%)   28 (73.7%)    116 (81.7%)    120    111 (92.5%)
+descriptions that are valid.                                             AnkiDroid      50      116      28 (56.0%)     107 (92.2%)    9 (32.1%)    93 (80.2%)     91     85 (93.4%)
+   We note that, because mobile apps lack precise functional speci-      OuterTune      41      76       41 (100.0%)    67 (88.2%)     9 (22.0%)    55 (72.4%)     48     44 (91.7%)
+
+fications, manual annotation in RQ1 mainly serves as a sanity check       NewPipe       46      106      45 (97.8%)     92 (86.8%)    20 (44.4%)    81 (76.4%)     79     72 (91.1%)
+                                                                        MaterialFiles   47      110      44 (93.6%)     103 (93.6%)   23 (52.3%)    82 (74.5%)     86     82 (95.3%)
+on the validity of the generated property descriptions. Whether            Orgzly       49      124      48 (98.0%)     115 (92.7%)   14 (29.2%)    88 (71.0%)     96     91 (94.8%)
+these properties are precise as executable specifications still needs     uhabits       55      80       38 (69.1%)     78 (97.5%)    16 (42.1%)    63 (78.8%)     69     64 (92.8%)
+
+to be validated through execution. Therefore, RQ2 further examines         Total        575    1282      491 (85.4%)   1210 (94.4%)   187 (35.2%)   977 (76.2%)    985    912 (92.6%)
+
+their precision by running the translated executable properties and
+analyzing the reported violations.
+Evaluation method of RQ2 and RQ3. RQ2 evaluates two aspects             analysis to determine whether each of the 25 bugs uncovered by
+of property refinement: (1) how many generated properties are im-       PropGen theoretically falls within the detection scope of each prior
+precise and thus lead to spurious violations during testing, and (2)    technique. This analysis is performed manually based on bug char-
+how many of these imprecise properties can be successfully refined      acteristics and the detection capabilities claimed by each technique.
+through refinement. RQ3 evaluates whether the generated proper-         To improve reliability, we further consulted the authors of Genie,
+ties can uncover new functional bugs. Specifically, for each app, we    Odin, and PBFDroid to validate our analysis. For VisionDroid, we
+load all initially generated executable properties into Kea [66] for    do not perform a separate scope analysis, since its LLM-based de-
+testing with 6 hours. Whenever Kea reports a property violation,        sign makes its theoretical detection scope difficult to characterize
+we manually inspect the property description, executable property       precisely; instead, we focus only on its empirical bug-finding per-
+code, violation-triggering execution trace, corresponding screen-       formance.
+shots and interaction events, and the assertion outcome. Based on          Second, we empirically evaluate each tool by running it on the
+this, we determine whether the violation is caused by a real app bug    corresponding apps and checking whether it can rediscover the
+or by non-bug factors, such as property imprecision or automation       same bugs in practice. For Genie, Odin, and VisionDroid, we fol-
+failures.                                                               low the default configurations described in their original papers.
+   For RQ2, we focus on the properties whose reported violations        PBFDroid requires users to manually specify properties for data
+are diagnosed as spurious. We apply refinement only to those caused     manipulation functionalities (DMFs). Therefore, we manually de-
+by property imprecision. For each such property, the refinement         fined the required DMF properties for detecting the corresponding
+module takes the original property description, executable property     bugs. To ensure fairness, we align the time budget with each tool’s
+code, and violation-triggering execution evidence as input, and         workflow: PropGen uses 3 hours for property generation and 6
+produces a revised executable property. We then re-execute the          hours for bug finding; accordingly, we allocate 9 hours per app to
+refined property on the app and inspect the result. A refinement        Genie, Odin, and VisionDroid, and 6 hours of automated testing to
+is considered successful if the revised property no longer triggers     PBFDroid after manual property construction.
+the same spurious violation while preserving the original testing
+intent. Based on this process, we report the number of properties       5.2         Results of RQ1
+sent to refinement due to property imprecision, the number and          Table 2 reports the results of functionality inference, execution,
+rate of successful refinements, and the breakdown of successfully       and property synthesis on all subject apps, where #Inferred Func
+refined properties by the modified component (i.e., precondition,       denotes the number of functionalities identified by tools, #Valid
+interaction, and postcondition).                                        Func, #Correctly Executed and #Valid Prop denote the numbers of
+   For RQ3, we focus on the violations diagnosed as real app bugs       functionalities and properties validated by human annotators, and
+through manual inspection. For each confirmed bug, we prepare           #Prop denotes the number of generated properties. For functionality-
+a bug report containing the bug description, reproduction steps,        related evaluation, we compare DroidAgent (D) and our approach
+expected and actual behaviors, and submit it to the corresponding       (P). Overall, our approach consistently outperforms DroidAgent
+app developers.                                                         in both functionality inference and execution. Across the 12 apps,
+Evaluation method of RQ4. RQ4 evaluates whether existing                our approach infers 1,282 functionalities, of which 1,210 are judged
+functional testing techniques can find the bugs uncovered by Prop-      valid, achieving 94.4% functionality validity, compared with 575
+Gen. We emphasize that this comparison is intended to assess            inferred functionalities and 491 valid ones (85.4%) for DroidAgent. It
+complementarity rather than replacement, i.e., whether PropGen          also correctly executes 977 functionalities, yielding 76.2% execution
+can uncover bugs that prior approaches may miss.                        correctness, substantially higher than DroidAgent’s 187 correctly
+   Following prior comparative analysis practice [66], we evalu-        executed functionalities and 35.2% execution correctness. On aver-
+ate these tools from two perspectives. First, we conduct a scope        age, this corresponds to 101 valid functionalities and 81 correctly
+From Exploration to Specification: LLM-Based Property Generation for Mobile App Testing                                       Conference’17, July 2017, Washington, DC, USA
+
+
+Table 3: Effectiveness of property refinement across subject                              Table 4: Statistics of the 25 new functional bugs found by the
+apps (RQ2).                                                                               generated properties.
+
+                                                               Modified Component          App Name        ID    Violated Property
+   App Name        #Imprecise Prop   #Successful Refinements
+                                                               Pre     I    Post                            1     Note info dialog should contain statistical data
+   OmniNotes             12                 11 (91.7%)         5      0      6                              2    The date should appear on the selection page
+    Markor               15                 12 (80.0%)         2      1      9                              3     The category selection should be changeable
+   RetroMusic            15                 14 (93.3%)         9      0      5                              4    The category page should be accessible from the drawer
+     Amaze               11                 10 (90.9%)         2      0      8                              5     The captured photo should appeared in the note content
+   MyExpenses            8                  8 (100.0%)         6      0      2             OmniNotes
+                                                                                                            6    Returning from the sketch should display the note title
+   AntennaPod            20                18 (90.0%)          7      0      11                             7     The reminder icon should be displayed after setting a reminder
+    AnkiDroid            9                 9 (100.0%)          7      0      2
+                                                                                                            8    The attachment should appear after selection
+    OuterTune            3                 3 (100.0%)          3      0      0
+                                                                                                            9     The image can be opened in the note content
+     NewPipe             11                10 (90.9%)          2      0      8
+                                                                                                           10    The FAB should appear after return
+   MaterialFiles         6                 6 (100.0%)          2      0      4
+                                                                                                            11    The specific item should disappear from the list
+      Orgzly             12                12 (100.0%)         5      0      7
+                                                                                                           12    The lyrics should be displayed after saving
+     uhabits             5                 5 (100.0%)          3      0      2
+                                                                                                            13    The item should not exist in the list
+      Total             127                118 (92.9%)         53     1      64            RetroMusic
+                                                                                                           14    The image should be displayed after change
+                                                                                                            15    The artist can be reset to default
+                                                                                                           16    The metadata should remain consistent
+                                                                                             Amaze          17    The item should be deleted successfully
+                                                                                                           18    The song list should open from navigation menu
+executed functionalities per app for our approach, compared with                            OuterTune
+                                                                                                            19    The file can be successfully created
+41 and 16, respectively, for DroidAgent.                                                   MaterialFiles   20    The item can be found
+   One likely reason for the performance gap is the difference in                                           21    Navigation to the sub-directory should succeed
+functionality inference. DroidAgent infers functionalities without                            uhabits      22    The added number should keep consistent
+explicitly grounding them to concrete GUI widgets, making some                                Orgzly       23     The imported file should be present
+inferred results loosely related to the current interface context and                         Markor       24    Star and "Favourite" checkbox should stay in sync
+                                                                                                           25    The insert image should be displayed after preview
+thus more likely to be invalid or non-executable. In contrast, our
+approach grounds functionality inference in the current GUI con-
+text, which helps produce more valid functionality hypotheses and                         apps: six apps achieve a 100% refinement rate, while the remaining
+makes subsequent execution more reliable. Also, we find DroidA-                           apps still achieve rates above 80%.
+gent tends to repeatedly execute failed events during functionality                          We further analyze which property components are modified
+execution.                                                                                during refinement. Among the 118 successfully refined properties,
+   For property synthesis, our approach generates 985 property                            53 involve precondition modifications, 64 involve postcondition
+descriptions, among which 912 are judged valid, corresponding to                          modifications, and only 1 involves an interaction modification. This
+92.6% property validity. Moreover, property validity exceeds 90% on                       suggests that most false positives can be resolved by refining when
+most apps, indicating that the synthesized properties are generally                       a property should be triggered or what outcome it should assert,
+well aligned with observed app behaviors.                                                 rather than changing the core interaction sequence. Among the
+LLM usage cost. For behavioral evidence construction and property                         127 refined properties, 9 remain not refined. We find that 5 of them
+synthesis, each LLM call consumes 6,198 tokens / $0.0145 on average.                      are caused by incorrect diagnosis of spurious violations, while the
+Overall, our approach uses 6,364k tokens / $14.86 per app on average.                     others occur because the revised property can no longer remain
+In comparison, DroidAgent uses 5,109k tokens / $13.01 per app on                          consistent with the original functionality. Note that for property
+average.                                                                                  refinement, each property requires 8,768 tokens / $0.02 on average.
+
+5.3      Results of RQ2                                                                   5.4     Results of RQ3
+Table 3 reports the effectiveness of our property refinement tech-                        Table 4 summarizes the bug-finding results, including the app name,
+nique across all subject apps, including how many generated prop-                         bug ID, and the brief description of the violated property. Overall,
+erties produce false positives during property-based testing, how                         PropGen uncovered 25 unique previously unknown functional
+many are successfully refined, and which property components are                          bugs in the latest app versions. Currently, 5 of the reported bugs
+modified.                                                                                 have been fixed by developers, while the remaining reports are
+    Among all generated executable properties, 127 produce spuri-                         waiting for responses.
+ous violations during testing and are thus identified as imprecise                           The discovered bugs cover diverse functionalities (e.g., note man-
+properties that lead to false positives. This result shows that prop-                     agement, attachment insertion, content display), suggesting that
+erty imprecision is not uncommon in LLM-generated properties,                             the generated properties can capture a broad range of behavioral
+and therefore refinement is necessary to improve their practical us-                      constraints in mobile apps. These bugs typically arise when the
+ability. Overall, our refinement technique is highly effective. Across                    actual app behavior deviates from the expected behavior encoded
+all apps, 127 properties are sent to refinement, and 118 of them are                      by the generated properties. For example, Bug 5 in OmniNotes is
+successfully refined, yielding an overall refinement rate of 92.9%.                       triggered by the property The captured photo should appear in the
+Moreover, the refinement performance is consistently strong across                        note content. During testing, it generates a GUI event sequence that
+Conference’17, July 2017, Washington, DC, USA                                                Yiheng Xiong, Shiwen Song, Bo Ma, Ting Su, and Xiaofei Xie
+
+
+Table 5: Results of prior functional testing tools for finding           7    Related Work
+the new functional bugs.
+                                                                         Automated mobile app GUI testing. Automated testing for mo-
+                                                                         bile apps has been extensively studied. Choudhary et al. [4] con-
+          Tool            #New Bugs in Scope    #New Bugs Found          ducted a systematic comparison of Android input-generation tools
+         Genie                  1 (4.0%)             0 (0.0%)
+          ODIN                  2 (8.0%)             0 (0.0%)
+                                                                         and highlighted both the promise and limitations of automated
+       PBFDroid                5 (17.9%)             2 (8.0%)            mobile testing. Prior work has proposed a variety of techniques
+      VisionDroid                   -                1 (4.0%)            to automatically explore app GUIs and generate event sequences
+          Total               7 (28.0%)             3 (12.0%)            for detecting crash bugs [8, 17, 29, 31, 33, 38, 47, 58, 61]. For ex-
+                                                                         ample, Sapienz [33] uses multi-objective search to generate event
+                                                                         sequences for improving coverage and exposing crashes. To find
+                                                                         non-crash functional bugs, most of prior work [1, 18, 42, 48, 50, 51,
+                                                                         60, 67, 71] designs automated oracles to overcome the oracle prob-
+first opens audio recording and then checks this property. After         lem. However, these work are limited to specific types of functional
+executing the corresponding interaction sequence, the captured           bugs (e.g., data losses [1, 18, 42, 71]). Some work leverages LLM to
+photo fails to appear in the note content, thus violating the prop-      analyze GUI pages during exploration to find data inconsistency
+erty. This bug is difficult to uncover through conventional manual       bugs [19], functional bugs [30], or inconsistencies between app
+testing, as testers typically focus on the main interaction path and     design and implementation [27].
+may not consider interleavings with other events.                           Property-based testing (PBT) is a powerful testing methodology
+                                                                         and have been adopted into many different software systems to
+                                                                         find logic bugs [2, 5, 20, 21, 23, 32, 35, 37, 43]. Recent work has
+5.5     Results of RQ4                                                   begun to bring property-based testing to mobile apps. Specifically,
+Table 5 summarizes how many of the new bugs found by PropGen             PBFDroid [49], PDTDroid [52], and Kea [66] have demonstrated
+can also be detected by Genie, Odin, PBFDroid, and VisionDroid.          that PBT can be effectively applied to GUI-driven mobile apps
+Among the 25 new bugs uncovered by PropGen, only 7 (28%) are             and can find non-crashing functional bugs that are difficult for
+within the scope of these prior techniques, and only 3 (12%) are         traditional automated GUI testing tools to detect. However, these
+actually found in practice.                                              work still assumes that meaningful properties are manually written
+    This result suggests that PropGen provides complementary bug-        by developers or testers. Our work complements these approaches
+finding capability to existing functional testing techniques. We         by automating the executable property generation.
+further analysis why most of the bugs cannot be found by prior           Automated test generation. Traditional automated test genera-
+techniques. Genie, Odin, and PBFDroid are designed with specific         tion techniques, such as fuzzing [34], symbolic execution [14, 45, 54],
+types of functional bugs, relying on predefined automated oracles or     and search-based testing [12, 36], mainly aim to improve coverage,
+manually specified DMF properties. As a result, they can only find       but often struggle to generate effective assertions [39, 46]. Learning-
+the bug categories emphasized in their original designs, whereas         based approaches leverage pre-trained language models to generate
+PropGen targets more broadly through generated properties. Vi-           tests from code [25, 55, 68, 70]. Recently, LLMs have shown strong
+sionDroid, in contrast, is a more general LLM-based functional           promise in test generation [3, 7, 9, 11, 13, 22, 25, 44, 59, 70]. Beyond
+testing approach. Its exploration strategy typically validates a func-   unit test generation, several recent studies have explored the use
+tionality by following one plausible interaction path at a time. In      of LLMs in property-related testing tasks. For example, prior work
+contrast, most bugs uncovered by PropGen do not appear on such           has investigated generating postconditions for individual functions
+a straightforward execution path. Instead, they are exposed only         from their comments [10], synthesizing property-based tests from
+when the property is checked under specific event sequences. In          specifications for Python libraries [57], and generating properties
+other words, these bugs are triggered not by whether the main func-      for smart contracts [28]. In contrast, our work investigates how
+tionality can be completed, but by whether the expected behavior         LLMs can be used to automatically generate properties for property-
+still holds under varied runtime conditions.                             based testing of mobile apps. Recently, different agents have been
+                                                                         proposed to automatically perform tasks on mobile apps [40, 41, 62–
+                                                                         64, 69, 72]. These works focus on executing user-provided tasks,
+6     Threats to Validity
+                                                                         whereas our work centers on automatically exploring app function-
+First, our evaluation involves manual inspection to assess the cor-      alities and generating executable properties.
+rectness of inferred functionalities and generated properties, which
+may introduce subjectivity and potential bias. To mitigate this
+threat, each case is independently labeled by two experienced grad-      8    Conclusion
+uate students following consistent evaluation criteria; disagree-        In this paper, we present PropGen, an automated approach for
+ments are further discussed until agreement is reached. Second, the      constructing properties of mobile apps. By exploring app function-
+apps used in our evaluation may not fully represent the diversity        alities and deriving properties from behavioral evidence, PropGen
+of real-world mobile apps. To mitigate this threat, most of the apps     reduces the need for manual property specification. We further
+are selected from prior relevant studies, and we further include         propose a feedback-driven refinement technique to refine impre-
+four additional apps to improve diversity. In the future, we plan to     cise properties exposed during testing. Experiments on real-world
+evaluate PropGen on a larger and broader set of apps.                    Android apps show that PropGen can effectively generate correct
+From Exploration to Specification: LLM-Based Property Generation for Mobile App Testing                                          Conference’17, July 2017, Washington, DC, USA
+
+
+executable properties. These results demonstrate the practical value                       [22] Zongze Jiang, Ming Wen, Jialun Cao, Xuanhua Shi, and Hai Jin. 2024. Towards
+of automated property construction for mobile app property-based                                Understanding the Effectiveness of Large Language Models on Directed Test
+                                                                                                Input Generation. In Proceedings of the 39th IEEE/ACM International Conference
+testing.                                                                                        on Automated Software Engineering. 1408–1420.
+                                                                                           [23] Stefan Karlsson, Adnan Čaušević, and Daniel Sundmark. 2020. QuickREST:
+                                                                                                Property-based test generation of OpenAPI-described RESTful APIs. In 2020
+References                                                                                      IEEE 13th International Conference on Software Testing, Validation and Verification
+ [1] Christoffer Quist Adamsen, Gianluca Mezzetti, and Anders Møller. 2015. System-             (ICST). IEEE, 131–141.
+     atic execution of android test suites in adverse conditions. In Proceedings of the    [24] Pavneet Singh Kochhar, Ferdian Thung, Nachiappan Nagappan, Thomas Zim-
+     2015 International Symposium on Software Testing and Analysis. 83–93.                      mermann, and David Lo. 2015. Understanding the test automation culture of
+ [2] Thomas Arts, John Hughes, Joakim Johansson, and Ulf Wiger. 2006. Testing                   app developers. In 2015 IEEE 8th International Conference on Software Testing,
+     telecoms software with Quviq QuickCheck. In Proceedings of the 2006 ACM                    Verification and Validation (ICST). IEEE, 1–10.
+     SIGPLAN Workshop on Erlang. 2–10.                                                     [25] Shuvendu K Lahiri, Sarah Fakhoury, Aaditya Naik, Georgios Sakkas, Saikat
+ [3] Yinghao Chen, Zehao Hu, Chen Zhi, Junxiao Han, Shuiguang Deng, and Jianwei                 Chakraborty, Madanlal Musuvathi, Piali Choudhury, Curtis von Veh, Jee-
+     Yin. 2024. Chatunitest: A framework for llm-based test generation. In Compan-              vana Priya Inala, Chenglong Wang, et al. 2022. Interactive code generation
+     ion Proceedings of the 32nd ACM International Conference on the Foundations of             via test-driven user-intent formalization. arXiv preprint arXiv:2208.05950 (2022).
+     Software Engineering. 572–576.                                                        [26] Mario Linares-Vásquez, Carlos Bernal-Cárdenas, Kevin Moran, and Denys Poshy-
+ [4] Shauvik Roy Choudhary, Alessandra Gorla, and Alessandro Orso. 2015. Au-                    vanyk. 2017. How do developers test android applications?. In 2017 IEEE In-
+     tomated test input generation for android: Are we there yet?(e). In 2015 30th              ternational Conference on Software Maintenance and Evolution (ICSME). IEEE,
+     IEEE/ACM International Conference on Automated Software Engineering (ASE).                 613–622.
+     IEEE, 429–440.                                                                        [27] Ruofan Liu, Xiwen Teoh, Yun Lin, Guanjie Chen, Ruofei Ren, Denys Poshyvanyk,
+ [5] Koen Claessen and John Hughes. 2000. QuickCheck: a lightweight tool for                    and Jin Song Dong. 2025. GUIPilot: A Consistency-Based Mobile GUI Testing
+     random testing of Haskell programs. In ICFP’00. 268–279.                                   Approach for Detecting Application-Specific Bugs. Proceedings of the ACM on
+ [6] Riccardo Coppola and Emil Alégroth. 2022. A taxonomy of metrics for GUI-                   Software Engineering 2, ISSTA (2025), 753–776.
+     based testing research: A systematic literature review. Information and Software      [28] Ye Liu, Yue Xue, Daoyuan Wu, Yuqiang Sun, Yi Li, Miaolei Shi, and Yang Liu.
+     Technology 152 (2022), 107062.                                                             2024. Propertygpt: Llm-driven formal verification of smart contracts through
+ [7] Yinlin Deng, Chunqiu Steven Xia, Haoran Peng, Chenyuan Yang, and Lingming                  retrieval-augmented property generation. arXiv preprint arXiv:2405.02580 (2024).
+     Zhang. 2023. Large language models are zero-shot fuzzers: Fuzzing deep-learning       [29] Zhe Liu, Chunyang Chen, Junjie Wang, Mengzhuo Chen, Boyu Wu, Xing Che,
+     libraries via large language models. In Proceedings of the 32nd ACM SIGSOFT                Dandan Wang, and Qing Wang. 2024. Make llm a testing expert: Bringing
+     international symposium on software testing and analysis. 423–435.                         human-like interaction to mobile gui testing via functionality-aware decisions. In
+ [8] Zhen Dong, Marcel Böhme, Lucia Cojocaru, and Abhik Roychoudhury. 2020.                     Proceedings of the IEEE/ACM 46th International Conference on Software Engineering.
+     Time-travel testing of android apps. In Proceedings of the ACM/IEEE 42nd Inter-            1–13.
+     national Conference on Software Engineering. 481–492.                                 [30] Zhe Liu, Cheng Li, Chunyang Chen, Junjie Wang, Mengzhuo Chen, Boyu Wu,
+ [9] Kohei Dozono, Tiago Espinha Gasiba, and Andrea Stocco. 2024. Large language                Yawen Wang, Jun Hu, and Qing Wang. 2025. Seeing is believing: Vision-driven
+     models for secure code assessment: A multi-language empirical study. arXiv                 non-crash functional bug detection for mobile apps. IEEE Transactions on Software
+     preprint arXiv:2408.06428 (2024).                                                          Engineering (2025).
+[10] Madeline Endres, Sarah Fakhoury, Saikat Chakraborty, and Shuvendu K Lahiri.           [31] Aravind Machiry, Rohan Tahiliani, and Mayur Naik. 2013. Dynodroid: An input
+     2024. Can large language models transform natural language intent into formal              generation system for android apps. In Proceedings of the 2013 9th Joint Meeting
+     method postconditions? Proceedings of the ACM on Software Engineering 1, FSE               on Foundations of Software Engineering. 224–234.
+     (2024), 1889–1912.                                                                    [32] David R MacIver, Zac Hatfield-Dodds, et al. 2019. Hypothesis: A new approach
+[11] Angela Fan, Beliz Gokkaya, Mark Harman, Mitya Lyubarskiy, Shubho Sengupta,                 to property-based testing. Journal of Open Source Software 4, 43 (2019), 1891.
+     Shin Yoo, and Jie M Zhang. 2023. Large language models for software engineering:      [33] Ke Mao, Mark Harman, and Yue Jia. 2016. Sapienz: Multi-objective automated
+     Survey and open problems. In 2023 IEEE/ACM International Conference on Software            testing for android applications. In Proceedings of the 25th international symposium
+     Engineering: Future of Software Engineering (ICSE-FoSE). IEEE, 31–53.                      on software testing and analysis. 94–105.
+[12] Gordon Fraser and Andrea Arcuri. 2011. Evosuite: automatic test suite generation      [34] Michał Zalewski. 2016. American Fuzzy Lop - Whitepaper. https://lcamtuf.
+     for object-oriented software. In Proceedings of the 19th ACM SIGSOFT symposium             coredump.cx/afl/technical_details.txt
+     and the 13th European conference on Foundations of software engineering. 416–419.     [35] Liam O’Connor and Oskar Wickström. 2022. Quickstrom: property-based accep-
+[13] Cuiyun Gao, Xing Hu, Shan Gao, Xin Xia, and Zhi Jin. 2025. The current chal-               tance testing with LTL specifications. In Proceedings of the 43rd ACM SIGPLAN
+     lenges of software engineering in the era of large language models. ACM Trans-             International Conference on Programming Language Design and Implementation
+     actions on Software Engineering and Methodology 34, 5 (2025), 1–30.                        (PLDI). 1025–1038. doi:10.1145/3519939.3523728
+[14] Patrice Godefroid, Nils Klarlund, and Koushik Sen. 2005. DART: Directed auto-         [36] Carlos Pacheco and Michael D Ernst. 2007. Randoop: feedback-directed random
+     mated random testing. In Proceedings of the 2005 ACM SIGPLAN conference on                 testing for Java. In Companion to the 22nd ACM SIGPLAN conference on Object-
+     Programming language design and implementation. 213–223.                                   oriented programming systems and applications companion. 815–816.
+[15] Harrison Goldstein, Joseph W Cutler, Daniel Dickstein, Benjamin C Pierce, and         [37] Rohan Padhye, Caroline Lemieux, and Koushik Sen. 2019. Jqf: Coverage-guided
+     Andrew Head. 2024. Property-based testing in practice. In Proceedings of the               property-based testing in java. In Proceedings of the 28th ACM SIGSOFT Interna-
+     IEEE/ACM 46th International Conference on Software Engineering. 1–13.                      tional Symposium on Software Testing and Analysis. 398–401.
+[16] Harrison Goldstein, Joseph W Cutler, Adam Stein, Benjamin C Pierce, and Andrew        [38] Minxue Pan, An Huang, Guoxin Wang, Tian Zhang, and Xuandong Li. 2020.
+     Head. 2022. Some problems with properties. In Proc. Workshop on the Human                  Reinforcement learning based curiosity-driven testing of android applications.
+     Aspects of Types and Reasoning Assistants (HATRA), Vol. 1. 3.                              In Proceedings of the 29th ACM SIGSOFT International Symposium on Software
+[17] Tianxiao Gu, Chengnian Sun, Xiaoxing Ma, Chun Cao, Chang Xu, Yuan Yao,                     Testing and Analysis. 153–164.
+     Qirun Zhang, Jian Lu, and Zhendong Su. 2019. Practical GUI testing of An-             [39] Annibale Panichella, Sebastiano Panichella, Gordon Fraser, Anand Ashok Sawant,
+     droid applications via model abstraction and refinement. In 2019 IEEE/ACM 41st             and Vincent J Hellendoorn. 2020. Revisiting test smells in automatically generated
+     International Conference on Software Engineering (ICSE). IEEE, 269–280.                    tests: limitations, pitfalls, and opportunities. In 2020 IEEE international conference
+[18] Wunan Guo, Zhen Dong, Liwei Shen, Wei Tian, Ting Su, and Xin Peng. 2022.                   on software maintenance and evolution (ICSME). IEEE, 523–533.
+     Detecting and fixing data loss issues in Android apps. In ISSTA ’22: 31st ACM         [40] Yujia Qin, Yining Ye, Junjie Fang, Haoming Wang, Shihao Liang, Shizuo Tian,
+     SIGSOFT International Symposium on Software Testing and Analysis. 605–616.                 Junda Zhang, Jiahao Li, Yunxin Li, Shijue Huang, et al. 2025. Ui-tars: Pioneering
+     doi:10.1145/3533767.3534402                                                                automated gui interaction with native agents. arXiv preprint arXiv:2501.12326
+[19] Yongxiang Hu, Hailiang Jin, Xuan Wang, Jiazhen Gu, Shiyu Guo, Chaoyi Chen,                 (2025).
+     Xin Wang, and Yangfan Zhou. 2024. Autoconsis: Automatic gui-driven data               [41] Dezhi Ran, Hao Wang, Zihe Song, Mengzhou Wu, Yuan Cao, Ying Zhang, Wei
+     inconsistency detection of mobile apps. In Proceedings of the 46th International           Yang, and Tao Xie. 2024. Guardian: A Runtime Framework for LLM-based UI
+     Conference on Software Engineering: Software Engineering in Practice. 137–146.             Exploration. In Proceedings of the 33rd ACM SIGSOFT International Symposium
+[20] John Hughes. 2016. Experiences with QuickCheck: testing the hard stuff and                 on Software Testing and Analysis.
+     staying sane. In A List of Successes That Can Change the World: Essays Dedicated      [42] Oliviero Riganelli, Simone Paolo Mottadelli, Claudio Rota, Daniela Micucci, and
+     to Philip Wadler on the Occasion of His 60th Birthday. Springer, 169–186.                  Leonardo Mariani. 2020. Data loss detector: automatically revealing data loss
+[21] John Hughes, Benjamin C Pierce, Thomas Arts, and Ulf Norell. 2016. Mysteries               bugs in Android apps. In ISSTA ’20: 29th ACM SIGSOFT International Symposium
+     of dropbox: property-based testing of a distributed synchronization service. In            on Software Testing and Analysis. 141–152. doi:10.1145/3395363.3397379
+     2016 IEEE International Conference on Software Testing, Verification and Validation
+     (ICST). IEEE, 135–145.
+Conference’17, July 2017, Washington, DC, USA                                                                      Yiheng Xiong, Shiwen Song, Bo Ma, Ting Su, and Xiaofei Xie
+
+
+[43] André Santos, Alcino Cunha, and Nuno Macedo. 2018. Property-based testing for         [65] Yiheng Xiong, Ting Su, Jingling Sun, Jue Wang, Qin Li, Geguang Pu, and Zhen-
+     the robot operating system. In Proceedings of the 9th ACM SIGSOFT International            dong Su. 2026. From Natural Language to Executable Properties for Property-
+     Workshop on Automating TEST Case Design, Selection, and Evaluation. 56–62.                 based Testing of Mobile Apps. arXiv:2603.21263 [cs.SE] https://arxiv.org/abs/
+[44] Max Schäfer, Sarah Nadi, Aryaz Eghbali, and Frank Tip. 2023. An empirical                  2603.21263
+     evaluation of using large language models for automated unit test generation.         [66] Yiheng Xiong, Ting Su, Jue Wang, Jingling Sun, Geguang Pu, and Zhendong
+     IEEE Transactions on Software Engineering 50, 1 (2023), 85–105.                            Su. 2024. General and Practical Property-based Testing for Android Apps. In
+[45] Koushik Sen, Darko Marinov, and Gul Agha. 2005. CUTE: A concolic unit testing              Proceedings of the 39th IEEE/ACM International Conference on Automated Software
+     engine for C. ACM SIGSOFT software engineering notes 30, 5 (2005), 263–272.                Engineering. 53–64.
+[46] Sina Shamshiri. 2015. Automated unit test generation for evolving software. In        [67] Yiheng Xiong, Mengqian Xu, Ting Su, Jingling Sun, Jue Wang, He Wen, Geguang
+     Proceedings of the 2015 10th Joint Meeting on Foundations of Software Engineering.         Pu, Jifeng He, and Zhendong Su. 2023. An empirical study of functional bugs in
+     1038–1041.                                                                                 android apps. In Proceedings of the 32nd ACM SIGSOFT International Symposium
+[47] Ting Su, Guozhu Meng, Yuting Chen, Ke Wu, Weiming Yang, Yao Yao, Geguang Pu,               on Software Testing and Analysis (ISSTA’23). 1319–1331.
+     Yang Liu, and Zhendong Su. 2017. Guided, Stochastic Model-based GUI Testing           [68] Lin Yang, Chen Yang, Shutao Gao, Weijing Wang, Bo Wang, Qihao Zhu, Xiao Chu,
+     of Android Apps. In The joint meeting of the European Software Engineering                 Jianyi Zhou, Guangtai Liang, Qianxiang Wang, et al. 2024. On the evaluation of
+     Conference and the ACM SIGSOFT Symposium on the Foundations of Software                    large language models in unit test generation. In Proceedings of the 39th IEEE/ACM
+     Engineering (ESEC/FSE). 245–256. doi:10.1145/3106237.3106298                               International Conference on Automated Software Engineering. 1607–1619.
+[48] Ting Su, Yichen Yan, Jue Wang, Jingling Sun, Yiheng Xiong, Geguang Pu, Ke             [69] Juyeon Yoon, Robert Feldt, and Shin Yoo. 2024. Intent-driven mobile gui test-
+     Wang, and Zhendong Su. 2021. Fully automated functional fuzzing of Android                 ing with autonomous large language model agents. In 2024 IEEE Conference on
+     apps for detecting non-crashing logic bugs. Proc. ACM Program. Lang. 5, OOPSLA             Software Testing, Verification and Validation (ICST). IEEE, 129–139.
+     (2021), 1–31. doi:10.1145/3485533                                                     [70] Zhiqiang Yuan, Mingwei Liu, Shiji Ding, Kaixin Wang, Yixuan Chen, Xin Peng,
+[49] Jingling Sun, Ting Su, Jiayi Jiang, Jue Wang, Geguang Pu, and Zhendong Su. 2023.           and Yiling Lou. 2024. Evaluating and improving chatgpt for unit test generation.
+     Property-Based Fuzzing for Finding Data Manipulation Errors in Android Apps.               Proceedings of the ACM on Software Engineering 1, FSE (2024), 1703–1726.
+     In Proceedings of the 31st ACM Joint European Software Engineering Conference         [71] Razieh Nokhbeh Zaeem, Mukul R. Prasad, and Sarfraz Khurshid. 2014. Automated
+     and Symposium on the Foundations of Software Engineering (ESEC/FSE). 1088–1100.            Generation of Oracles for Testing User-Interaction Features of Mobile Apps. In
+     doi:10.1145/3611643.3616286                                                                Proceedings of the International Conference on Software Testing, Verification and
+[50] Jingling Sun, Ting Su, Junxin Li, Zhen Dong, Geguang Pu, Tao Xie, and Zhendong             Validation (ICST). 183–192. doi:10.1109/ICST.2014.31
+     Su. 2021. Understanding and finding system setting-related defects in Android         [72] Chi Zhang, Zhao Yang, Jiaxuan Liu, Yanda Li, Yucheng Han, Xin Chen, Zebiao
+     apps. In ISSTA ’21: 30th ACM SIGSOFT International Symposium on Software                   Huang, Bin Fu, and Gang Yu. 2025. Appagent: Multimodal agents as smartphone
+     Testing and Analysis. 204–215. doi:10.1145/3460319.3464806                                 users. In Proceedings of the 2025 CHI Conference on Human Factors in Computing
+[51] Jingling Sun, Ting Su, Kai Liu, Chao Peng, Zhao Zhang, Geguang Pu, Tao Xie,                Systems. 1–20.
+     and Zhendong Su. 2023. Characterizing and Finding System Setting-Related
+     Defects in Android Apps. IEEE Trans. Software Eng. 49, 4 (2023), 2941–2963.
+     doi:10.1109/TSE.2023.3236449
+[52] Jingling Sun, Ting Su, Jun Sun, Jianwen Li, Mengfei Wang, and Geguang Pu.
+     2024. Property-Based Testing for Validating User Privacy-Related Functionalities
+     in Social Media Apps. In Companion Proceedings of the 32nd ACM International
+     Conference on the Foundations of Software Engineering. 440–451.
+[53] Android Team. 2021. Android Debug Bridge (adb). Retrieved 2026-3 from https:
+     //developer.android.com/tools/adb
+[54] Nikolai Tillmann, Jonathan De Halleux, and Tao Xie. 2014. Transferring an
+     automated test generation tool to practice: From Pex to Fakes and Code Digger. In
+     Proceedings of the 29th ACM/IEEE International Conference on Automated Software
+     Engineering. 385–396.
+[55] Michele Tufano, Dawn Drain, Alexey Svyatkovskiy, Shao Kun Deng, and Neel
+     Sundaresan. 2020. Unit test case generation with transformers and focal context.
+     arXiv preprint arXiv:2009.05617 (2020).
+[56] uiautomator2 Team. 2021. uiautomator2. Retrieved 2026-3 from https://github.
+     com/openatx/uiautomator2
+[57] Vasudev Vikram, Caroline Lemieux, Joshua Sunshine, and Rohan Padhye. 2023.
+     Can large language models write good property-based tests? arXiv preprint
+     arXiv:2307.04346 (2023).
+[58] Chenxu Wang, Tianming Liu, Yanjie Zhao, Minghui Yang, and Haoyu Wang.
+     2025. Llmdroid: Enhancing automated mobile app gui testing coverage with
+     large language model guidance. Proceedings of the ACM on Software Engineering
+     2, FSE (2025), 1001–1022.
+[59] Junjie Wang, Yuchao Huang, Chunyang Chen, Zhe Liu, Song Wang, and Qing
+     Wang. 2024. Software testing with large language models: Survey, landscape,
+     and vision. IEEE Transactions on Software Engineering 50, 4 (2024), 911–936.
+[60] Jue Wang, Yanyan Jiang, Ting Su, Shaohua Li, Chang Xu, Jian Lu, and Zhendong
+     Su. 2022. Detecting non-crashing functional bugs in Android apps via deep-
+     state differential analysis. In Proceedings of the 30th ACM Joint European Software
+     Engineering Conference and Symposium on the Foundations of Software Engineering
+     (ESEC/FSE). 434–446. doi:10.1145/3540250.3549170
+[61] Jue Wang, Yanyan Jiang, Chang Xu, Chun Cao, Xiaoxing Ma, and Jian Lu. 2020.
+     Combodroid: generating high-quality test inputs for android apps via use case
+     combinations. In Proceedings of the ACM/IEEE 42nd International Conference on
+     Software Engineering. 469–480.
+[62] Junyang Wang, Haiyang Xu, Haitao Jia, Xi Zhang, Ming Yan, Weizhou Shen, Ji
+     Zhang, Fei Huang, and Jitao Sang. 2024. Mobile-agent-v2: Mobile device operation
+     assistant with effective navigation via multi-agent collaboration. arXiv preprint
+     arXiv:2406.01014 (2024).
+[63] Hao Wen, Yuanchun Li, Guohong Liu, Shanhui Zhao, Tao Yu, Toby Jia-Jun Li, Shiqi
+     Jiang, Yunhao Liu, Yaqin Zhang, and Yunxin Liu. 2024. Autodroid: Llm-powered
+     task automation in android. In Proceedings of the 30th Annual International Con-
+     ference on Mobile Computing and Networking. 543–557.
+[64] Hao Wen, Hongming Wang, Jiaxuan Liu, and Yuanchun Li. 2023. Droidbot-gpt:
+     Gpt-powered ui automation for android. arXiv preprint arXiv:2304.07061 (2023).
+

packages/opencode/specs/simulation.md

@@ -0,0 +1,438 @@
+# Opencode Simulation Architecture
+
+Status: first milestone architecture draft.
+
+## Goal
+
+Build a simulation environment for exploring opencode through the real app, primarily through the TUI, while replacing only the lowest foundational layers needed to make runs controlled, observable, and safe.
+
+The first milestone is an interactive exploration and model-based testing environment. It should be enough to start opencode normally, put the app into generated states, drive real user-level TUI actions, observe what happened, and record an in-memory trace that can later be exported into deterministic replay tests.
+
+This is not intended to be a custom simulated app or a separate `simulate` command. The normal app should run, with simulation enabled by one required flag:
+
+```sh
+OPENCODE_SIMULATION=1 bun run dev
+```
+
+## Non-Goals
+
+- Do not reimplement the app.
+- Do not replace mid-level services like session processing, tool registry, provider orchestration, route trees, or TUI components unless a foundational seam proves impossible.
+- Do not build shrinking in the first milestone.
+- Do not make generated randomized runs part of CI yet.
+- Do not build differential testing in the first milestone.
+- Do not expose simulation controls when `OPENCODE_SIMULATION` is not set.
+
+## Design Principles
+
+- Run the real app through normal commands.
+- Drive the TUI using real user-level input: typing, keypresses, focus, click, and mouse actions.
+- Keep simulation code isolated under a simulation/testing area.
+- Touch production app code only at narrow activation points: builders, TUI startup, foundational layers, and simulation-gated backend routes.
+- Swap foundational layers, not app logic.
+- Make observations rich enough for humans and models.
+- Treat traces as first-class artifacts.
+- Use a lightweight model of expected high-level behavior, not a clone of opencode internals.
+- Generate valid commands from current observed state rather than blindly fuzzing impossible actions.
+
+## Activation
+
+`OPENCODE_SIMULATION=1` is the only required flag.
+
+Optional flags can be added later, but should stay minimal. Reasonable optional parameters later include renderer mode, trace output path, seed, or port override.
+
+When enabled:
+
+- The app builds with simulation layer replacements.
+- The TUI process starts a loopback WebSocket control server.
+- Simulation-gated backend control routes become available only to the frontend/control path.
+- In-memory trace recording starts automatically.
+
+## Control Server
+
+The external control surface lives in the TUI/frontend process, not the backend API server.
+
+This is important because the frontend has direct access to the renderer, screen state, focus state, interactable elements, and user input APIs. The backend remains the normal backend, with only simulation-gated control routes used internally by the frontend when needed.
+
+Protocol:
+
+- JSON-RPC 2.0 over WebSocket.
+- Loopback only.
+- Start at `127.0.0.1:40900`.
+- If occupied, scan upward and report the actual URL.
+- External drivers connect only to this frontend WebSocket.
+
+The app should not send JSON-RPC requests back to the driver in the first milestone. The driver sends requests; the app responds and emits notifications/events as useful.
+
+Initial method groups:
+
+- `ui.state`: return screen, elements, focus, and generated possible actions.
+- `ui.action`: execute one real user-level action.
+- `ui.render`: force or wait for a render and return state.
+- `backend.filesystem.seed`: seed project files.
+- `backend.filesystem.write`: write one file.
+- `backend.network.register`: register a fake network response.
+- `backend.llm.enqueue`: queue scripted LLM behavior.
+- `backend.snapshot`: return backend simulation state.
+- `trace.list`: return trace records.
+- `trace.clear`: clear in-memory trace.
+- `trace.export`: export trace JSON for replay/test generation.
+- `run.stabilize`: wait for frontend/backend quiescence and return observations.
+
+## TUI Actions
+
+The old simulation branch had the right basic shape: observe OpenTUI renderables, derive executable actions, and execute those actions through OpenTUI input/mouse APIs.
+
+The first action vocabulary should stay close to that work:
+
+```ts
+type UIAction =
+  | { type: "typeText"; text: string }
+  | { type: "pressKey"; key: string; modifiers?: KeyModifiers }
+  | { type: "pressEnter" }
+  | { type: "pressArrow"; direction: "up" | "down" | "left" | "right" }
+  | { type: "focus"; target: number }
+  | { type: "click"; target: number; x: number; y: number }
+```
+
+`ui.state` should return:
+
+- Current screen text.
+- Focused renderable/editor state.
+- Interactable elements.
+- Generated actions valid for the current UI state.
+
+Elements should include stable-enough semantic data where available:
+
+- Renderable ID and numeric target.
+- Position and dimensions.
+- Focusable/clickable/editor flags.
+- Focused flag.
+- Text or label when available.
+- Role/capability when available.
+
+Both fake OpenTUI renderer and visible terminal renderer should share this protocol. The architecture should support both; the default can be decided later.
+
+## Backend Control
+
+The backend server should be exactly the normal backend server.
+
+Simulation-only backend routes may exist, but only when `OPENCODE_SIMULATION=1`. They are private implementation details for the frontend simulation server to proxy commands like filesystem seeding, LLM scripting, network registration, and snapshots.
+
+External drivers should not use backend simulation routes directly.
+
+## Foundational Layer Replacement
+
+Current `origin/dev` has the right seam: `AppNodeBuilder.build(...)` and `AppNodeBuilderV1.build(...)` accept replacements over `LayerNode`s. Simulation should use those seams instead of adding large alternate app assemblies.
+
+First milestone replacements:
+
+- Filesystem.
+- Network / HTTP client.
+- LLM boundary.
+- Process spawner.
+
+First milestone generated state surfaces:
+
+- Filesystem/project state.
+- Network responses.
+- LLM scripts.
+- Process registry behavior.
+- Plugin-generated config state.
+
+Likely later replacements:
+
+- Clock/random.
+- Database path/isolation.
+- Global paths/temp paths.
+
+The goal is to swap things at the bottom of the app. Everything above these foundational services should behave as production code.
+
+## Filesystem
+
+The first filesystem simulation should be temp-directory backed.
+
+Rationale:
+
+- It is easier to inspect while debugging.
+- It avoids reimplementing all filesystem semantics immediately.
+- It makes generated scenarios concrete and replayable.
+- It keeps the path open to tool behavior that expects real files.
+
+The temp filesystem is still controlled and isolated:
+
+- Each run gets its own temp root.
+- Project files, config, data, state, cache, and temp paths should resolve inside that root.
+- Host filesystem escapes should fail loudly.
+- Trace should record seeded files and file diffs/observations needed for replay.
+
+## Configuration Via Generated Plugins
+
+Generated configuration is a core first-milestone feature.
+
+Much of opencode behavior is driven by config. The simulation runner needs to put the app into many different config-shaped states: different agents, tools, providers, MCP servers, permissions, modes, instructions, formatting settings, feature flags, and other config-dependent behavior.
+
+The runner should not primarily generate arbitrary config files. Instead, the simulation should express config-shaped state as generated plugins.
+
+Rationale:
+
+- Plugins are already a normal extension surface for opencode behavior.
+- Generated plugins can produce app states without making the simulation depend on config-file syntax and file layout details.
+- Plugin-generated state keeps setup closer to runtime behavior: the app reads config, loads plugins, and observes plugin-provided behavior through normal app paths.
+- Plugins are a better unit for model-based generation because they can be named, versioned, traced, reused, and minimized independently.
+
+The first implementation should support generated simulation plugins that can contribute or affect config-equivalent domains such as:
+
+- Agents and agent defaults.
+- Provider/model availability.
+- Tool definitions and tool behavior.
+- MCP-like capabilities or endpoints.
+- Permission defaults and policies.
+- Instructions/system-context-like inputs where supported.
+- Formatting/project behavior where supported.
+- Workspace/project adapters where supported.
+
+The simulation can still write the minimal bootstrap state needed for opencode to discover generated plugins, but the interesting generated state should live in plugin definitions rather than large generated `opencode.json` files.
+
+Trace should record:
+
+- Generated plugin IDs.
+- Plugin-provided config/state fragments.
+- Plugin hooks registered.
+- Any plugin load/config errors.
+- Which generated plugin state was active for each run.
+
+The model-based runner should include commands for generating and enabling plugin state. These commands should have normal preconditions and postconditions just like UI actions or backend setup commands.
+
+Example command families:
+
+- Generate a provider/model plugin.
+- Generate an agent configuration plugin.
+- Generate a tool plugin with scripted behavior.
+- Generate permission policy state.
+- Generate MCP-like tool/resource state.
+- Enable or disable a generated plugin for the next app run.
+
+This is the main mechanism for exploring app states driven by configuration.
+
+## Network
+
+Unknown external network should fail loudly by default.
+
+The simulation network should support explicit response registration:
+
+- JSON response.
+- Text response.
+- Bytes response later if needed.
+- Status-only response.
+- Handler-style response later if needed.
+
+Loopback traffic needed by the app/frontend/backend may be allowed explicitly.
+
+All network calls should be traceable:
+
+- Method.
+- URL.
+- Request headers/body where safe.
+- Matched simulation route.
+- Status.
+- Response summary.
+- Error if denied.
+
+## LLM
+
+The LLM boundary should be scriptable.
+
+The driver can enqueue scripts that describe model behavior:
+
+- Text chunks.
+- Thinking/reasoning chunks if relevant.
+- Tool calls.
+- Errors.
+- Finish reason.
+
+The real session and tool pipeline should consume this behavior through the normal app path. The simulation should not bypass `SessionPrompt`, `SessionProcessor`, or tool execution.
+
+Missing scripted LLM behavior should fail with a clear simulation error unless a default response is explicitly configured.
+
+## Process Spawning
+
+External process spawning should be denied by default.
+
+The first milestone should provide a simulated process registry. This should be inspired by the old branch:
+
+- Shell commands can run through `just-bash` against the simulated filesystem.
+- A small fake `git` command set can support project discovery/status paths needed by the app.
+- Unsupported process spawns fail loudly.
+
+This preserves the rule that simulation does not spawn arbitrary external programs while still allowing useful shell/tool flows.
+
+## Trace
+
+Trace recording is always on in simulation mode, in memory for the first milestone.
+
+Trace entries should be append-only JSON-compatible records. They do not need to be written to disk initially, but `trace.export` should return a structure suitable for later replay and test generation.
+
+Trace should include:
+
+- Run metadata: seed, app version, renderer mode, WebSocket URL.
+- Initial world setup.
+- UI observations.
+- Generated UI actions.
+- Executed UI actions.
+- Backend control requests.
+- Backend snapshots.
+- Network requests and matches/denials.
+- LLM scripts enqueued and consumed.
+- Tool calls and results.
+- Permission decisions.
+- Filesystem seed/write/diff summaries.
+- Generated plugin/config state and load results.
+- Stabilization boundaries.
+- Errors and crashes.
+- Model command execution and postcondition results.
+
+The trace is the bridge between exploratory simulation and deterministic tests.
+
+## Model-Based Runner
+
+The first runner is an external driver connecting to the frontend WebSocket.
+
+Use a custom runner for now, not `fast-check`. It should still follow the core shape used by property/model-based testing libraries:
+
+```ts
+interface Command<Model> {
+  readonly name: string
+  check(model: Model): boolean
+  run(model: Model, app: SimulationClient): Promise<void>
+}
+```
+
+Basic runner responsibilities:
+
+- Keep a lightweight model of high-level expected state.
+- Generate commands whose preconditions match the model and current app observations.
+- Execute commands through the WebSocket.
+- Update the model.
+- Check postconditions/invariants.
+- Record all steps in the trace.
+- Support seed/replay.
+- Track simple distribution stats.
+
+The model should track high-level, observational state only, such as:
+
+- Current screen/route category.
+- Whether prompt editor is available.
+- Known sessions.
+- Known files and expected file contents/diffs.
+- Queued LLM scripts.
+- Recent backend/session status.
+- Whether app is expected to be idle.
+
+The model must not track implementation internals like fibers, exact runner loop state, cache internals, or database implementation details.
+
+Initial command families:
+
+- Seed filesystem.
+- Generate and enable plugin config state.
+- Register network response.
+- Enqueue LLM script.
+- Observe UI state.
+- Execute one generated UI action.
+- Type prompt text.
+- Press enter.
+- Stabilize.
+- Assert no crash.
+- Assert visible response or file effect.
+- Export trace.
+
+## Generators
+
+The first milestone should include generation, but not shrinking.
+
+Generation should be model-based and state-aware:
+
+- Generate from currently valid `ui.state.actions`.
+- Generate backend setup commands from scenario/model state.
+- Generate plugin-provided config state.
+- Generate LLM scripts that match likely user prompts and tool flows.
+- Generate short command sequences using preconditions.
+- Use a seed so runs can be replayed.
+- Use simple weights to avoid degenerate action selection.
+
+The generator should not attempt to produce arbitrary full app states upfront. It should build state by executing commands through the real app and observing the result.
+
+Important stats to record:
+
+- Seed.
+- Command counts.
+- Action type distribution.
+- Generated plugin/config domain distribution.
+- Rejected command/precondition counts.
+- UI element/action coverage.
+- Backend event type coverage where available.
+- Errors and stabilization failures.
+
+## Properties
+
+First milestone properties should be simple and high-signal:
+
+- App does not crash.
+- Backend does not crash.
+- Unknown network is denied.
+- Host filesystem escape is denied.
+- Prompt submission can reach a scripted LLM response.
+- Stabilization eventually reaches a coherent idle state for the demo flow.
+- File effects from scripted tool behavior are observable in the simulated filesystem.
+- Trace contains enough information to replay the run.
+
+More advanced model/refinement, metamorphic, and differential properties are future work.
+
+## First Demo Flow
+
+The first major demo should show this system as a real environment for exploring the app in controlled states:
+
+1. Start opencode normally with `OPENCODE_SIMULATION=1`.
+2. TUI starts and exposes the simulation WebSocket on `127.0.0.1:40900+`.
+3. External runner connects.
+4. Runner seeds a temp-backed project filesystem.
+5. Runner generates and enables plugin-provided config state.
+6. Runner queues a scripted LLM response.
+7. Runner observes `ui.state` and generated actions.
+8. Runner drives real TUI input to type and submit a prompt.
+9. App processes the prompt through the real backend/session/tool path.
+10. Scripted LLM response appears or executes a file-affecting tool flow.
+11. Runner stabilizes the app.
+12. Runner inspects trace, backend snapshot, UI state, generated plugin state, and filesystem state.
+13. Runner exports a deterministic replay trace.
+
+## Done-When Checklist
+
+- `OPENCODE_SIMULATION=1` starts the normal app with simulation wiring.
+- Simulation code is isolated under a dedicated simulation/testing area.
+- App changes outside simulation are limited to activation hooks, builder replacements, TUI startup, and gated backend routes.
+- TUI exposes JSON-RPC WebSocket on `127.0.0.1:40900+`.
+- Driver can call `ui.state`.
+- Driver can execute generated UI actions.
+- Fake and visible renderer paths use the same action protocol.
+- Driver can seed filesystem state.
+- Driver can generate and enable plugin-provided config state.
+- Driver can register network responses and observe denied unknown network.
+- Driver can enqueue LLM scripts.
+- External process spawning is denied by default, with shell via `just-bash` and minimal fake process registry support.
+- Driver can run a basic model-based generated command sequence.
+- In-memory trace records observations/actions/backend interactions.
+- Driver can list, clear, and export trace.
+- Demo flow succeeds end-to-end.
+
+## Future Directions
+
+- Shrinking failed traces.
+- Promote minimized traces into normal committed tests.
+- Coverage-guided corpus and structured trace mutation.
+- Richer semantic UI grounding for model-driven exploration.
+- LLM-generated property proposals with validity/soundness checks.
+- Differential testing across app versions, renderers, or storage modes.
+- Deterministic scheduler/clock/random control.
+- Parallel campaigns with isolated workers.
+- File-backed trace persistence and replay CLI.

packages/opencode/src/cli/cmd/run/runtime.lifecycle.ts

@@ -228,7 +228,6 @@ export async function createRuntimeLifecycle(input: LifecycleInput): Promise<Lif
     const { RunFooter } = await footerTask
     let closed = false
     let sigintRegistered = false
-
     const footer = new RunFooter(renderer, {
       directory: input.directory,
       findFiles: input.findFiles,

packages/tui/src/app.tsx

@@ -230,6 +230,17 @@ export const run = Effect.fn("Tui.run")(function* (input: TuiInput) {
       renderer.once("destroy", () => Deferred.doneUnsafe(shutdown, Effect.void))
       const pluginRuntime = createPluginRuntime()
 
+      const simulation = yield* Effect.promise(async () => {
+        if (process.env.OPENCODE_SIMULATION !== "1" && process.env.OPENCODE_SIMULATION !== "true") return
+        const { SimulationActions } = await import("./simulation/actions")
+        const { SimulationServer } = await import("./simulation/server")
+        return SimulationServer.start(SimulationActions.createHarness(renderer))
+      })
+      if (simulation) {
+        process.stderr.write(`opencode simulation websocket: ${simulation.url}\n`)
+        yield* Effect.addFinalizer(() => Effect.sync(() => simulation.stop()))
+      }
+
       yield* Effect.tryPromise(async () => {
         // Prewarm palette before ThemeProvider mounts so `system` theme avoids a first-paint fallback flash.
         void renderer.getPalette({ size: 16 }).catch(() => undefined)

packages/tui/src/simulation/actions.ts

@@ -0,0 +1,161 @@
+import type { CliRenderer, Renderable } from "@opentui/core"
+import { createMockKeys, createMockMouse } from "@opentui/core/testing"
+import { SimulationTrace } from "./trace"
+
+export interface KeyModifiers {
+  readonly ctrl?: boolean
+  readonly shift?: boolean
+  readonly meta?: boolean
+  readonly super?: boolean
+  readonly hyper?: boolean
+}
+
+export type Action =
+  | { readonly type: "typeText"; readonly text: string }
+  | { readonly type: "pressKey"; readonly key: string; readonly modifiers?: KeyModifiers }
+  | { readonly type: "pressEnter" }
+  | { readonly type: "pressArrow"; readonly direction: "up" | "down" | "left" | "right" }
+  | { readonly type: "focus"; readonly target: number }
+  | { readonly type: "click"; readonly target: number; readonly x: number; readonly y: number }
+
+export interface Element {
+  readonly id: string
+  readonly num: number
+  readonly x: number
+  readonly y: number
+  readonly width: number
+  readonly height: number
+  readonly focusable: boolean
+  readonly focused: boolean
+  readonly clickable: boolean
+  readonly editor: boolean
+}
+
+export interface Harness {
+  readonly renderer: CliRenderer
+  readonly renderOnce: () => Promise<void>
+  readonly screen: () => string
+}
+
+type RenderBuffer = {
+  readonly width: number
+  readonly height: number
+  getRealCharBytes(includeAnsi?: boolean): Uint8Array
+}
+
+const decoder = new TextDecoder()
+
+function children(renderable: Renderable) {
+  return renderable.getChildren().filter((child): child is Renderable => "num" in child)
+}
+
+function all(renderable: Renderable): Renderable[] {
+  return [renderable, ...children(renderable).flatMap(all)]
+}
+
+function mouseListeners(renderable: Renderable) {
+  const general = Reflect.get(renderable, "_mouseListener")
+  const specific = Reflect.get(renderable, "_mouseListeners")
+  return Boolean(general) || (specific && typeof specific === "object" && Object.keys(specific).length > 0)
+}
+
+function hit(renderer: CliRenderer, renderable: Renderable) {
+  if (renderable.width <= 0 || renderable.height <= 0) return false
+  const x = Math.floor(renderable.screenX + renderable.width / 2)
+  const y = Math.floor(renderable.screenY + renderable.height / 2)
+  return renderer.hitTest(x, y) === renderable.num
+}
+
+export function createHarness(renderer: CliRenderer): Harness {
+  return {
+    renderer,
+    renderOnce: async () => {
+      renderer.requestRender()
+      await renderer.idle()
+    },
+    screen: () => decoder.decode((Reflect.get(renderer, "currentRenderBuffer") as RenderBuffer).getRealCharBytes(true)),
+  }
+}
+
+export function elements(renderer: CliRenderer): Element[] {
+  return all(renderer.root)
+    .filter((renderable) => renderable.visible && !renderable.isDestroyed)
+    .map((renderable) => {
+      const clickable = mouseListeners(renderable) && hit(renderer, renderable)
+      return {
+        id: renderable.id,
+        num: renderable.num,
+        x: renderable.screenX,
+        y: renderable.screenY,
+        width: renderable.width,
+        height: renderable.height,
+        focusable: renderable.focusable,
+        focused: renderable.focused,
+        clickable,
+        editor: renderer.currentFocusedEditor === renderable,
+      } satisfies Element
+    })
+    .filter((element) => element.focusable || element.clickable || element.editor)
+}
+
+export function actions(renderer: CliRenderer, options: { text?: string } = {}): Action[] {
+  const items = elements(renderer)
+  return [
+    ...(renderer.currentFocusedEditor
+      ? ([{ type: "typeText", text: options.text ?? "hello" }, { type: "pressEnter" }] satisfies Action[])
+      : []),
+    ...items.filter((item) => item.focusable && !item.focused).map((item) => ({ type: "focus" as const, target: item.num })),
+    ...items
+      .filter((item) => item.clickable)
+      .map((item) => ({
+        type: "click" as const,
+        target: item.num,
+        x: Math.floor(item.x + item.width / 2),
+        y: Math.floor(item.y + item.height / 2),
+      })),
+    { type: "pressArrow", direction: "down" },
+    { type: "pressArrow", direction: "up" },
+  ]
+}
+
+export function state(harness: Harness) {
+  return {
+    screen: harness.screen(),
+    focused: {
+      renderable: harness.renderer.currentFocusedRenderable?.num,
+      editor: Boolean(harness.renderer.currentFocusedEditor),
+    },
+    elements: elements(harness.renderer),
+    actions: actions(harness.renderer),
+  }
+}
+
+export async function execute(harness: Harness, action: Action) {
+  const mockInput = createMockKeys(harness.renderer)
+  const mockMouse = createMockMouse(harness.renderer)
+  SimulationTrace.add("ui.action", { action })
+  switch (action.type) {
+    case "typeText":
+      await mockInput.typeText(action.text)
+      break
+    case "pressKey":
+      mockInput.pressKey(action.key, action.modifiers)
+      break
+    case "pressEnter":
+      mockInput.pressEnter()
+      break
+    case "pressArrow":
+      mockInput.pressArrow(action.direction)
+      break
+    case "focus":
+      all(harness.renderer.root).find((item) => item.num === action.target)?.focus()
+      break
+    case "click":
+      await mockMouse.click(action.x, action.y)
+      break
+  }
+  await harness.renderOnce()
+  return state(harness)
+}
+
+export * as SimulationActions from "./actions"

packages/tui/src/simulation/server.ts

@@ -0,0 +1,170 @@
+import { SimulationActions, type Action, type Harness } from "./actions"
+import { SimulationTrace } from "./trace"
+
+const DefaultPort = 40900
+const MaxPortAttempts = 100
+
+type JsonRpcRequest = {
+  readonly jsonrpc: "2.0"
+  readonly id?: string | number | null
+  readonly method: string
+  readonly params?: unknown
+}
+
+type JsonRpcResponse = {
+  readonly jsonrpc: "2.0"
+  readonly id: string | number | null
+  readonly result?: unknown
+  readonly error?: {
+    readonly code: number
+    readonly message: string
+    readonly data?: unknown
+  }
+}
+
+export interface Server {
+  readonly url: string
+  readonly stop: () => void
+}
+
+function isEnabled() {
+  return process.env.OPENCODE_SIMULATION === "1" || process.env.OPENCODE_SIMULATION === "true"
+}
+
+function isPortUnavailable(error: unknown) {
+  const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase()
+  return message.includes("eaddrinuse") || message.includes("address already in use") || message.includes(" in use")
+}
+
+function parseRequest(input: string | Buffer): JsonRpcRequest {
+  const value = JSON.parse(typeof input === "string" ? input : input.toString()) as unknown
+  if (typeof value !== "object" || value === null) throw new Error("Invalid JSON-RPC request")
+  if (!("jsonrpc" in value) || value.jsonrpc !== "2.0") throw new Error("Invalid JSON-RPC version")
+  if (!("method" in value) || typeof value.method !== "string") throw new Error("Invalid JSON-RPC method")
+  return value as JsonRpcRequest
+}
+
+function isAction(input: unknown): input is Action {
+  if (typeof input !== "object" || input === null || !("type" in input)) return false
+  switch (input.type) {
+    case "typeText":
+      return "text" in input && typeof input.text === "string"
+    case "pressKey":
+      return "key" in input && typeof input.key === "string"
+    case "pressEnter":
+      return true
+    case "pressArrow":
+      return "direction" in input && ["up", "down", "left", "right"].includes(String(input.direction))
+    case "focus":
+      return "target" in input && typeof input.target === "number"
+    case "click":
+      return (
+        "target" in input &&
+        typeof input.target === "number" &&
+        "x" in input &&
+        typeof input.x === "number" &&
+        "y" in input &&
+        typeof input.y === "number"
+      )
+  }
+  return false
+}
+
+function actionParam(params: unknown) {
+  if (typeof params !== "object" || params === null || !("action" in params)) throw new Error("Missing action")
+  if (!isAction(params.action)) throw new Error("Invalid action")
+  return params.action
+}
+
+function response(id: JsonRpcRequest["id"], result: unknown): JsonRpcResponse | undefined {
+  if (id === undefined) return undefined
+  return { jsonrpc: "2.0", id, result }
+}
+
+function errorResponse(id: JsonRpcRequest["id"], error: unknown): JsonRpcResponse {
+  return {
+    jsonrpc: "2.0",
+    id: id ?? null,
+    error: {
+      code: -32000,
+      message: error instanceof Error ? error.message : String(error),
+    },
+  }
+}
+
+async function handle(harness: Harness, request: JsonRpcRequest) {
+  switch (request.method) {
+    case "ui.state": {
+      const result = SimulationActions.state(harness)
+      SimulationTrace.add("ui.state", { elements: result.elements.length, actions: result.actions.length })
+      return result
+    }
+    case "ui.action":
+      return SimulationActions.execute(harness, actionParam(request.params))
+    case "ui.render": {
+      await harness.renderOnce()
+      const result = SimulationActions.state(harness)
+      SimulationTrace.add("ui.render", { elements: result.elements.length, actions: result.actions.length })
+      return result
+    }
+    case "trace.list":
+      return { records: SimulationTrace.list() }
+    case "trace.clear":
+      SimulationTrace.clear()
+      return { cleared: true }
+    case "trace.export":
+      return SimulationTrace.exportTrace()
+  }
+  throw new Error(`Unknown simulation method: ${request.method}`)
+}
+
+function serve(harness: Harness, port = DefaultPort, attempts = MaxPortAttempts): Bun.Server {
+  try {
+    return Bun.serve<{ readonly simulation: true }>({
+      hostname: "127.0.0.1",
+      port,
+      fetch(request, server) {
+        if (server.upgrade(request, { data: { simulation: true } })) return undefined
+        return new Response("opencode simulation websocket", { status: 426 })
+      },
+      websocket: {
+        open() {
+          SimulationTrace.add("control.connect")
+        },
+        close() {
+          SimulationTrace.add("control.disconnect")
+        },
+        async message(socket, message) {
+          let request: JsonRpcRequest | undefined
+          try {
+            request = parseRequest(message)
+            const result = await handle(harness, request)
+            const next = response(request.id, result)
+            if (next) socket.send(JSON.stringify(next))
+          } catch (error) {
+            socket.send(JSON.stringify(errorResponse(request?.id, error)))
+          }
+        },
+      },
+    })
+  } catch (error) {
+    if (!isPortUnavailable(error) || attempts <= 1 || port >= 65535) throw error
+    return serve(harness, port + 1, attempts - 1)
+  }
+}
+
+export function start(harness: Harness): Server | undefined {
+  if (!isEnabled()) return
+  const server = serve(harness)
+  const url = `ws://${server.hostname}:${server.port}`
+  SimulationTrace.add("control.start", { url })
+  return {
+    url,
+    stop: () => {
+      SimulationTrace.add("control.stop", { url })
+      server.stop(true)
+    },
+  }
+}
+
+export * as SimulationServer from "./server"

packages/tui/src/simulation/trace.ts

@@ -0,0 +1,37 @@
+export type TraceRecord = {
+  readonly id: number
+  readonly time: string
+  readonly type: string
+  readonly data?: unknown
+}
+
+const records: TraceRecord[] = []
+let nextID = 0
+
+export function add(type: string, data?: unknown) {
+  const record = {
+    id: ++nextID,
+    time: new Date().toISOString(),
+    type,
+    ...(data === undefined ? {} : { data }),
+  } satisfies TraceRecord
+  records.push(record)
+  return record
+}
+
+export function list() {
+  return [...records]
+}
+
+export function clear() {
+  records.length = 0
+  nextID = 0
+}
+
+export function exportTrace() {
+  return {
+    records: list(),
+  }
+}
+
+export * as SimulationTrace from "./trace"