diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a90d53d..1e101a30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,13 @@ Release discipline: direct substrate-layout or `git-warp` runtime leakage - removed Think-managed `git-warp` cache/checkpoint reads, doctor checks, and ref deletion from product runtime paths -- upgraded `@git-stunts/git-warp` to 18.2.1 +- replaced the production storage path with the public + `@git-stunts/git-warp` v19 Runtime, Lane, Intent, Observer, and captured + coordinate APIs, with no v18 compatibility path +- upgraded the registry dependency to `@git-stunts/git-warp` 19.0.2 +- read bounded native index pages concurrently at one captured coordinate, + preserving deterministic newest-first recall while avoiding serial optic + latency ## [0.7.2] - 2026-06-23 diff --git a/benchmarks/capture-latency.js b/benchmarks/capture-latency.js index edc5fb19..e62cf48a 100644 --- a/benchmarks/capture-latency.js +++ b/benchmarks/capture-latency.js @@ -123,7 +123,7 @@ function measureCapturePhases(homeDir, text) { const wallMs = Number(wallEnd - wallStart) / 1_000_000; - const events = result.stderr + const events = `${result.stdout}\n${result.stderr}` .split('\n') .filter((line) => line.trim().startsWith('{')) .map((line) => { diff --git a/contracts/think-git-warp-v19.graphql b/contracts/think-git-warp-v19.graphql new file mode 100644 index 00000000..f1527c9c --- /dev/null +++ b/contracts/think-git-warp-v19.graphql @@ -0,0 +1,120 @@ +# Think's authored git-warp v19 application contract. +# +# Wesley owns deterministic operation metadata. Think's renderer binds that +# metadata to git-warp's public v19 Intent and Observer constructors. +# +# This is intentionally not a generic graph or property facade. The only +# retained property shapes exposed to application code are Think's three +# schema-versioned byte documents. + +directive @intent( + kind: IntentKind! + subject: String + property: String + value: String +) on FIELD_DEFINITION + +directive @observer( + id: String! + reading: ReadingKind! + subject: String! + property: String + cardinality: ObserverCardinality! + decoder: ValueDecoder! +) on FIELD_DEFINITION + +enum IntentKind { + NODE_ADD + PROPERTY_SET_BYTES +} + +enum ReadingKind { + PROPERTY + NODE_EXISTS +} + +enum ObserverCardinality { + EXACTLY_ONE +} + +enum ValueDecoder { + BOOLEAN + BYTES +} + +type IntentBinding { + operation: String! +} + +scalar Bytes + +type Mutation { + declareMemoryObject(subject: ID!): IntentBinding! + @intent(kind: NODE_ADD, subject: "subject") + + storeMemoryDocument(subject: ID!, value: Bytes!): IntentBinding! + @intent( + kind: PROPERTY_SET_BYTES + subject: "subject" + property: "think.memory-document.v1" + value: "value" + ) + + storeMemoryIndex(subject: ID!, value: Bytes!): IntentBinding! + @intent( + kind: PROPERTY_SET_BYTES + subject: "subject" + property: "think.memory-index.v1" + value: "value" + ) + + storeMemoryIndexPage(subject: ID!, value: Bytes!): IntentBinding! + @intent( + kind: PROPERTY_SET_BYTES + subject: "subject" + property: "think.memory-index-page.v1" + value: "value" + ) + +} + +type Query { + memoryDocument(subject: ID!): Bytes + @observer( + id: "think.memory.document" + reading: PROPERTY + subject: "subject" + property: "think.memory-document.v1" + cardinality: EXACTLY_ONE + decoder: BYTES + ) + + memoryIndex(subject: ID!): Bytes + @observer( + id: "think.memory.index" + reading: PROPERTY + subject: "subject" + property: "think.memory-index.v1" + cardinality: EXACTLY_ONE + decoder: BYTES + ) + + memoryIndexPage(subject: ID!): Bytes + @observer( + id: "think.memory.index-page" + reading: PROPERTY + subject: "subject" + property: "think.memory-index-page.v1" + cardinality: EXACTLY_ONE + decoder: BYTES + ) + + memoryObjectExists(subject: ID!): Boolean! + @observer( + id: "think.memory.object-exists" + reading: NODE_EXISTS + subject: "subject" + cardinality: EXACTLY_ONE + decoder: BOOLEAN + ) +} diff --git a/docs/MIND_ORCHESTRATION.md b/docs/MIND_ORCHESTRATION.md index 8a69ffc0..9c199047 100644 --- a/docs/MIND_ORCHESTRATION.md +++ b/docs/MIND_ORCHESTRATION.md @@ -11,17 +11,25 @@ Any directory under `~/.think/` that contains a git repository (a `.git/` subdirectory) is a mind. The directory name is the mind's display name. +The repository must contain Think's native git-warp v19 application data. +Think does not open v18 substrates or the rejected v19 compatibility-record +layout. Migrate those repositories with +`docs/operations/git-warp-v19-cutover.md` before placing them at an +authoritative mind path. + ``` ~/.think/ - repo/ → "default" mind (the original single-mind path) + james/ → "default" human mind + repo/ → "repo" mind (retained pre-James path, when present) claude/ → "claude" mind work/ → "work" mind metrics/ → NOT a mind (no .git/) ``` -The special directory `~/.think/repo` displays as **"default"** for -backward compatibility — it's the mind Think uses when no other is -selected. +The special directory `~/.think/james` displays as **"default"** and +is the mind Think uses when no other is selected. A retained +`~/.think/repo` remains discoverable under the display name **"repo"**; +Think does not silently merge or delete it. ## Creating a mind @@ -32,8 +40,9 @@ git init ``` That's it. Think discovers it automatically on the next browse launch. -No configuration files, no registration step. The filesystem is the -registry. +The first capture initializes the native v19 substrate and bounded Think +indexes. No configuration file or separate registration command is required; +the filesystem is the registry. ## Discovery @@ -72,7 +81,7 @@ The header shows the active mind name when multiple minds exist ## Capture -Capture always goes to the default mind (`~/.think/repo`) or +Capture always goes to the default mind (`~/.think/james`) or whatever `THINK_REPO_DIR` points to. Mind selection in browse is read-only — it does not change which mind receives new captures. @@ -89,7 +98,7 @@ both capture and browse. The mind switcher in the TUI is limited to a single-element list containing the overridden path. When `THINK_REPO_DIR` is not set, Think discovers all minds under -`~/.think/` and uses `~/.think/repo` as the default. +`~/.think/` and uses `~/.think/james` as the default. ## Agent isolation diff --git a/docs/audit/hexagonal-boundary-ratchet-baseline.json b/docs/audit/hexagonal-boundary-ratchet-baseline.json index 434d75d6..8fc6bcf8 100644 --- a/docs/audit/hexagonal-boundary-ratchet-baseline.json +++ b/docs/audit/hexagonal-boundary-ratchet-baseline.json @@ -1,13 +1,17 @@ { "allowedBoundaryFiles": [ "scripts/hexagonal-boundary-ratchet.mjs", + "scripts/prepare-v19-mind.mjs", + "scripts/replay-v19-capture-on-v18.mjs", "scripts/repair-v17-mind.mjs", "src/browse-benchmark.js", "src/browse/adapters/git-warp-worker.js", "src/browse/adapters/git-warp.js", "src/cli/commands/doctor.js", "src/doctor.js", - "src/history/git-warp-read.js" + "src/history/git-warp-read.js", + "src/store/git-warp-v19.js", + "src/store/think-warp-sdk.js" ], "generatedFrom": "scripts/hexagonal-boundary-ratchet.mjs", "leaks": { @@ -19,14 +23,13 @@ "src/store/enrichment/runner.js": 2, "src/store/migrations.js": 6, "src/store/reflect.js": 6, - "src/store/runtime.js": 11 + "src/store/runtime.js": 4 }, "byTerm": { - "git-warp-package": 1, - "git-warp-runtime-symbol": 37, + "git-warp-runtime-symbol": 31, "warp-ref-layout": 1 }, - "total": 39 + "total": 32 }, "substrateTermIds": [ "git-warp-package", diff --git a/docs/method/backlog/bad-code/CORE_git-warp-dependency-truth.md b/docs/method/backlog/bad-code/CORE_git-warp-dependency-truth.md index 73051c82..b776b8e8 100644 --- a/docs/method/backlog/bad-code/CORE_git-warp-dependency-truth.md +++ b/docs/method/backlog/bad-code/CORE_git-warp-dependency-truth.md @@ -10,8 +10,9 @@ blocked_by: Think runtime compatibility must be proven against the published `@git-stunts/git-warp` package declared in `package.json`, not against a linked local checkout or private package internals. The current product runtime targets -the public worldline API in `@git-stunts/git-warp@18.2.1`; legacy checkpoint -repair remains quarantined in the v17 repair lane. +the public Runtime, Lane, Intent, Observer, and captured-coordinate APIs in +`@git-stunts/git-warp@19.0.2`; legacy checkpoint repair remains quarantined in +the v17 repair lane. ## Acceptance Criteria diff --git a/docs/method/backlog/bad-code/CORE_large-mind-read-timeouts.md b/docs/method/backlog/bad-code/CORE_large-mind-read-timeouts.md index 9102a5bc..8f51f3dc 100644 --- a/docs/method/backlog/bad-code/CORE_large-mind-read-timeouts.md +++ b/docs/method/backlog/bad-code/CORE_large-mind-read-timeouts.md @@ -25,6 +25,11 @@ unavailable even when the underlying mind is intact. not call `read.view.query()`. - [x] Store self-contained fast capture records so default `remember` can score ambient project matches without hydrating capture nodes. +- [x] Read the bounded native index page set concurrently at one captured v19 + coordinate while preserving newest-first order. Six installed-wrapper + runs on `~/.think/codex` reduced median `codex-think --remember --json` + wall time from 11.111s to 4.814s at `/Users/james` and from 11.161s to + 5.081s at `/Users/james/git/think`. - [ ] Add a deterministic large-mind fixture or synthetic benchmark for MCP read timeout budgets. - [ ] Add an explicit read-model backfill/repair command for existing minds so @@ -32,8 +37,9 @@ unavailable even when the underlying mind is intact. from genesis. - [ ] Establish target budgets for `recent`, `stats`, `doctor`, and `remember` against large repaired minds. Current `codex-think --remember --json` - smoke exits under a 20s timeout on `~/.think/codex`, but still spends - roughly 13-14s in one git-warp optic property read. + exits reliably on `~/.think/codex`, but its 4.814-5.081s median remains + above the sub-second product doctrine; the measured scan phase still + consumes about 3.7s for nine bounded v19 index-page reads. - [ ] Document and automate safe maintenance for high-loose-object minds. -- [ ] Prefer public worldline/optic bounded reads where broad transitional +- [x] Prefer public worldline/optic bounded reads where broad transitional queries are not required. diff --git a/docs/operations/git-warp-v19-cutover.md b/docs/operations/git-warp-v19-cutover.md new file mode 100644 index 00000000..3d5405ce --- /dev/null +++ b/docs/operations/git-warp-v19-cutover.md @@ -0,0 +1,206 @@ +# Think Native git-warp v19 Cutover + +This runbook replaces one authoritative Think mind with a repository written +only through Think's native git-warp v19 SDK. + +The production runtime has no v18 reader and no compatibility representation. +It stores schema-versioned Think aggregate pages and their bounded indexes in +fixed byte-valued v19 properties. User object identity, chronology, sessions, +annotations, and reflection relationships are facts inside those immutable +page documents; they are not duplicated into one v19 commit per object and +edge. The rejected `think.record.v1` representation is readable only by the +one-time `convert-v19-mind.mjs` source converter. + +## Preconditions + +- Install Think's locked official `@git-stunts/git-warp` package. +- Stop every process that can write to the mind. +- Keep the maintenance window closed until post-swap verification passes. +- Use independent copies. A normal clone fetches branch refs only and is not a + faithful Think copy unless every `refs/*` namespace is fetched explicitly. +- Do not run Git garbage collection during the campaign. + +## Per-mind sequence + +Set paths explicitly for one mind: + +```bash +SOURCE="$HOME/.think/codex" +BACKUP="$HOME/.think-v19-cutover//backups/codex.pre-native-v19.git" +LEGACY_COPY="$HOME/.think-v19-cutover//sources/codex" +NATIVE_CANDIDATE="$HOME/.think-v19-cutover//candidates/codex" +REPORTS="$HOME/.think-v19-cutover//reports" +INVENTORY="$REPORTS/codex.native-v19.inventory.json" +``` + +1. Record every authoritative ref outside the repository: + + ```bash + git -C "$SOURCE" for-each-ref \ + --format='%(refname) %(objectname)' \ + > "$REPORTS/codex.refs.before.txt" + ``` + +2. Create and verify an independent mirror backup: + + ```bash + git clone --mirror --no-hardlinks "$SOURCE" "$BACKUP" + git -C "$BACKUP" fsck --full + ``` + +3. Create a non-hardlinked normal source copy, then fetch all refs without a + force refspec: + + ```bash + git clone --no-hardlinks "$SOURCE" "$LEGACY_COPY" + git -C "$LEGACY_COPY" fetch --no-tags "$SOURCE" 'refs/*:refs/*' + git -C "$LEGACY_COPY" fsck --full + ``` + +4. Prove the source copy is exact at the ref level: + + ```bash + git -C "$SOURCE" for-each-ref \ + --format='%(refname) %(objectname)' | + LC_ALL=C sort | shasum -a 256 + + git -C "$LEGACY_COPY" for-each-ref \ + --format='%(refname) %(objectname)' | + LC_ALL=C sort | shasum -a 256 + ``` + + The two digests must match. Dangling objects reported by `git fsck` are not + corruption; missing or corrupt objects are. + +5. Read the legacy application representation exactly once through one public + v19 observer plan and persist a checksummed inventory: + + ```bash + npm run migrate:v19-native -- \ + --source "$LEGACY_COPY" \ + --inventory-out "$INVENTORY" \ + --json + ``` + + The converter snapshots all source refs before and after the read, aborts if + they moved, and creates the inventory with exclusive-create semantics. It + never overwrites an existing inventory. Record the reported source-ref and + manifest SHA-256 values. + +6. Independently validate the persisted inventory without reopening the legacy + repository: + + ```bash + npm run migrate:v19-native -- \ + --inventory-in "$INVENTORY" \ + --dry-run \ + --json \ + > "$REPORTS/codex.native-inventory-verification.json" + ``` + +7. Initialize an empty normal Git repository for the native candidate: + + ```bash + mkdir -p "$NATIVE_CANDIDATE" + git init "$NATIVE_CANDIDATE" + git -C "$NATIVE_CANDIDATE" config core.fsmonitor false + git -C "$NATIVE_CANDIDATE" config user.name think + git -C "$NATIVE_CANDIDATE" config user.email think@local.invalid + ``` + +8. Import the immutable inventory into the empty native candidate. This phase + does not open or read the legacy repository: + + ```bash + npm run migrate:v19-native -- \ + --inventory-in "$INVENTORY" \ + --target "$NATIVE_CANDIDATE" \ + --json \ + > "$REPORTS/codex.native-conversion.json" + ``` + + The report must say `converted` and `verified: true`. The converter refuses + a target that already contains refs and rejects an inventory whose manifest, + counts, or per-kind totals do not match its content. + + The inventory is lossless, but the native projection deliberately does not + reproduce implementation artifacts. It retains user-authored entries, + sessions, annotations, unmatched historical thoughts, and their relationship + facts inside bounded aggregate pages. It drops: + + - `read_model:*` cached indexes; + - `artifact:*` derived receipts and projections; + - generated keyword, topic, and classification nodes; + - pipeline-run records; + - graph metadata initialized by the native runtime; and + - thought documents duplicated by a retained capture's `thoughtId`; and + - legacy graph edges whose meaning is already present in retained document + facts or bounded index order. + + The conversion report includes the full source summary plus imported and + dropped document/edge counts. Review those counts before continuing. The + complete inventory, source mirror, and pre-cutover backup remain the + evidence and recovery boundary for everything omitted from the new + authority. + +9. Verify candidate integrity and native runtime behavior: + + - `git fsck --full` succeeds. + - A bounded recent read returns the expected latest captures and count. + - Exact document samples match the conversion report. + - `think.record.v1` is absent from the candidate's admitted application + properties. + - Exact document samples resolve from the same bounded aggregate pages used + by list reads; there is no duplicate per-document property. + - A separate disposable clone of the candidate accepts a capture, returns + its Receipt, closes, reopens, and returns that capture. + +10. Create and verify a second independent mirror of the finished candidate. + Re-read the authoritative ref snapshot. Abort if any authoritative ref + moved after the maintenance window began. + +11. Atomically replace an existing authoritative directory with the compiled + `scripts/atomic-swap-paths.swift` helper. The old authoritative directory + lands at the candidate path and remains an immediate rollback source. + +12. If the requested authoritative path does not exist, publish with the + compiled `scripts/atomic-publish-path.swift` helper. It uses + `RENAME_EXCL`, atomically creates the target, and refuses to overwrite a + path created by another process. + +13. Through the authoritative path, repeat integrity, bounded-read, close, + reopen, and backup verification. Start writers only after all checks pass. + +## James source + +`~/.think/repo` is the retained James source and must remain unchanged. +Prepare and migrate a disposable copy with the official v19.0.2 substrate +migrator before using it as `--source`. Publish the native result to the absent +`~/.think/james` path. Never rename or overwrite `~/.think/repo`. + +## Required order + +Complete the entire backup, convert, verify, atomic replacement, and +post-replacement verification cycle in this order: + +1. `~/.think/codex` +2. `~/.think/claude` +3. `~/.think/gemini` +4. `~/.think/james` + +Do not begin the next authority while the previous one is unverified. + +## Rollback + +Stop writers before rollback. + +For an existing path, run the same atomic directory swap again. This restores +the exact pre-cutover repository as one filesystem operation. Do not rebuild a +rollback by moving individual refs or copying object files into a live +repository. + +For a newly published target, atomically move the failed publication back to +its absent candidate path. The original retained source remains unchanged. + +Keep the pre-cutover mirror, swapped-out directory, ref snapshots, conversion +reports, and post-conversion mirror until retention is reviewed separately. diff --git a/package-lock.json b/package-lock.json index 86d0fc51..8f062c45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "@flyingrobots/bijou-node": "7.1.0", "@flyingrobots/bijou-tui": "7.1.0", "@git-stunts/alfred": "0.10.3", - "@git-stunts/git-warp": "^18.2.1", + "@git-stunts/git-warp": "^19.0.2", "@git-stunts/plumbing": "3.0.3", "@modelcontextprotocol/sdk": "^1.29.0", "zod": "^4.3.6" @@ -358,6 +358,7 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/@git-stunts/git-cas/-/git-cas-6.0.0.tgz", "integrity": "sha512-NyTOaCHq6VBGCbL6HKR0bmX3uarumLAR+s2R8pofMGC3WX3YaS1pNdwTJOOzpvcZGWu3FKWAUVU9U0rdEyRoaw==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@flyingrobots/bijou": "^5.0.0", @@ -382,6 +383,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/@flyingrobots/bijou/-/bijou-5.0.0.tgz", "integrity": "sha512-Vmcs1jZYIxwb2NOn+LCDMK8ZmIKz64eTQI+gEk11Odn32s4ipIrzawrfrrAWZ4UTsdD5c9xWwwJH6SYgo5klBg==", + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=18" @@ -391,6 +393,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/@flyingrobots/bijou-i18n/-/bijou-i18n-5.0.0.tgz", "integrity": "sha512-S3HUHBBLh7fZlijcyuJvtrcJYa+rlhmfW5AAaMZmvIS1P9yd38t7P3JaPGsmKPJjCIVsVgiH3zrjWY15TKB6xA==", + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=18" @@ -400,6 +403,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/@flyingrobots/bijou-node/-/bijou-node-5.0.0.tgz", "integrity": "sha512-Ano8ydJKF/M8MGPeQDMjOuLouKFEOnEaTwYOKAHHMWxDh03G0Yt9HqZPPu364puzJuyFkY1APxtsxUCbru+5IA==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@flyingrobots/bijou-tui": "5.0.0", @@ -418,6 +422,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/@flyingrobots/bijou-tui/-/bijou-tui-5.0.0.tgz", "integrity": "sha512-dJAWBIZ8osXIM5y6Mc2KsawHPYR/3JxQEO78yVzzdsQiAx9PZLTQvB8fY6oUQRf+/n3sU9l2Ep0UUsJIbhAmpA==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@flyingrobots/bijou-i18n": "5.0.0" @@ -433,23 +438,27 @@ "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" } }, "node_modules/@git-stunts/git-warp": { - "version": "18.2.1", - "resolved": "https://registry.npmjs.org/@git-stunts/git-warp/-/git-warp-18.2.1.tgz", - "integrity": "sha512-M84jOC3Uukio+DlX6ARbCzpl4lfyCB9eodN5pR1hmzHkOSIJyaGCaYPPAafcnSTUYUwUd3z2p3rA9Sq3WoJeOQ==", + "version": "19.0.2", + "resolved": "https://registry.npmjs.org/@git-stunts/git-warp/-/git-warp-19.0.2.tgz", + "integrity": "sha512-nmmiH7tmVaZMejGCtnCsu57t7scvMqyhfMFpCajJ37B6eI5zTaX/6ruNj6Ik4HX5qC2NqcYeokc9u7Zee2VPSg==", "license": "Apache-2.0", "workspaces": [ "packages/*" ], "dependencies": { + "@flyingrobots/bijou": "^7.2.0", + "@flyingrobots/bijou-node": "^7.2.0", + "@flyingrobots/bijou-tui": "^7.2.0", "@git-stunts/alfred": "^0.10.4", - "@git-stunts/git-cas": "^6.0.0", - "@git-stunts/plumbing": "^3.0.3", + "@git-stunts/git-cas": "^6.5.5", + "@git-stunts/plumbing": "^3.2.0", "@git-stunts/trailer-codec": "^2.1.1", "@noble/hashes": "^2.2.0", "boxen": "^7.1.1", @@ -465,12 +474,63 @@ }, "bin": { "git-warp": "bin/git-warp", - "warp-graph": "dist/bin/warp-graph.js" + "git-warp-v18-to-v19": "dist/scripts/v18-to-v19/migrate.js" }, "engines": { "node": ">=22.0.0" } }, + "node_modules/@git-stunts/git-warp/node_modules/@flyingrobots/bijou": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@flyingrobots/bijou/-/bijou-7.2.0.tgz", + "integrity": "sha512-f7Ik7Wx/DBgaVoplt5QmrFiH84ri1p1DIfyYpxkcCAEwbhf3i+IU3bd3rohWaHWXtC7J5FINOmW8Cgbo9SVSuw==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@git-stunts/git-warp/node_modules/@flyingrobots/bijou-i18n": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@flyingrobots/bijou-i18n/-/bijou-i18n-7.2.0.tgz", + "integrity": "sha512-1c2DPftoSxBRfp7EDpNknRpoQsODbjyy3SMY1soRSU4C61lEX/FM3P1mtbO5HmYRkYuzBolgWZhLZ7e1BMP5KA==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@git-stunts/git-warp/node_modules/@flyingrobots/bijou-node": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@flyingrobots/bijou-node/-/bijou-node-7.2.0.tgz", + "integrity": "sha512-8PCWqr3WsBclZAnTqtnNd5xi85VFiCpbVjVrJ+jbb3UhC6QShz0meREUjSaSdfdguol9tribuy4a7hZaiqsvRw==", + "license": "Apache-2.0", + "dependencies": { + "@flyingrobots/bijou-tui": "7.2.0", + "chalk": "^5.6.2", + "gifenc": "^1.0.3", + "oled-font-5x7": "^1.0.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@flyingrobots/bijou": "7.2.0" + } + }, + "node_modules/@git-stunts/git-warp/node_modules/@flyingrobots/bijou-tui": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@flyingrobots/bijou-tui/-/bijou-tui-7.2.0.tgz", + "integrity": "sha512-cwr4Lh38toZCt+DDCAkaxcJn7DLST0fZDVfbTL8CRmtO9611SxvDqmw8Ij18reEmY63G72qVXlBsyfza7QhKdg==", + "license": "Apache-2.0", + "dependencies": { + "@flyingrobots/bijou-i18n": "7.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@flyingrobots/bijou": "7.2.0" + } + }, "node_modules/@git-stunts/git-warp/node_modules/@git-stunts/alfred": { "version": "0.10.4", "resolved": "https://registry.npmjs.org/@git-stunts/alfred/-/alfred-0.10.4.tgz", @@ -480,6 +540,95 @@ "node": ">=20.0.0" } }, + "node_modules/@git-stunts/git-warp/node_modules/@git-stunts/git-cas": { + "version": "6.5.5", + "resolved": "https://registry.npmjs.org/@git-stunts/git-cas/-/git-cas-6.5.5.tgz", + "integrity": "sha512-x2ohvIq04o5W3eFmn/x6WQ8UuXcqIxcdKuQOhttZJ80ZokuY+97xqqi7cJCDdrmeJMkbcuSW1VvWFzcz6K6TAg==", + "license": "Apache-2.0", + "dependencies": { + "@flyingrobots/bijou": "^5.0.0", + "@flyingrobots/bijou-node": "^5.0.0", + "@flyingrobots/bijou-tui": "^5.0.0", + "@flyingrobots/bijou-tui-app": "^5.0.0", + "@git-stunts/alfred": "^0.10.0", + "@git-stunts/plumbing": "^3.2.0", + "@git-stunts/vault": "^1.0.1", + "cbor-x": "^1.6.0", + "commander": "14.0.3", + "zod": "^3.24.1" + }, + "bin": { + "git-cas": "bin/git-cas.js" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@git-stunts/git-warp/node_modules/@git-stunts/git-cas/node_modules/@flyingrobots/bijou": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@flyingrobots/bijou/-/bijou-5.0.0.tgz", + "integrity": "sha512-Vmcs1jZYIxwb2NOn+LCDMK8ZmIKz64eTQI+gEk11Odn32s4ipIrzawrfrrAWZ4UTsdD5c9xWwwJH6SYgo5klBg==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@git-stunts/git-warp/node_modules/@git-stunts/git-cas/node_modules/@flyingrobots/bijou-i18n": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@flyingrobots/bijou-i18n/-/bijou-i18n-5.0.0.tgz", + "integrity": "sha512-S3HUHBBLh7fZlijcyuJvtrcJYa+rlhmfW5AAaMZmvIS1P9yd38t7P3JaPGsmKPJjCIVsVgiH3zrjWY15TKB6xA==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@git-stunts/git-warp/node_modules/@git-stunts/git-cas/node_modules/@flyingrobots/bijou-node": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@flyingrobots/bijou-node/-/bijou-node-5.0.0.tgz", + "integrity": "sha512-Ano8ydJKF/M8MGPeQDMjOuLouKFEOnEaTwYOKAHHMWxDh03G0Yt9HqZPPu364puzJuyFkY1APxtsxUCbru+5IA==", + "license": "Apache-2.0", + "dependencies": { + "@flyingrobots/bijou-tui": "5.0.0", + "chalk": "^5.6.2", + "gifenc": "^1.0.3", + "oled-font-5x7": "^1.0.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@flyingrobots/bijou": "5.0.0" + } + }, + "node_modules/@git-stunts/git-warp/node_modules/@git-stunts/git-cas/node_modules/@flyingrobots/bijou-tui": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@flyingrobots/bijou-tui/-/bijou-tui-5.0.0.tgz", + "integrity": "sha512-dJAWBIZ8osXIM5y6Mc2KsawHPYR/3JxQEO78yVzzdsQiAx9PZLTQvB8fY6oUQRf+/n3sU9l2Ep0UUsJIbhAmpA==", + "license": "Apache-2.0", + "dependencies": { + "@flyingrobots/bijou-i18n": "5.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@flyingrobots/bijou": "5.0.0" + } + }, + "node_modules/@git-stunts/git-warp/node_modules/@git-stunts/plumbing": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@git-stunts/plumbing/-/plumbing-3.2.0.tgz", + "integrity": "sha512-wW3Rzq6KNX0iygU9LpF8odQpcAyylH0KrUxNTDeRJHnwxfIamm7DWBL8lDkP1TqltE+4Bi0H0rjyPtvSkAJJeg==", + "license": "Apache-2.0", + "dependencies": { + "zod": "^3.24.1" + }, + "engines": { + "bun": ">=1.3.5", + "deno": ">=2.0.0", + "node": ">=20.0.0" + } + }, "node_modules/@git-stunts/git-warp/node_modules/zod": { "version": "3.24.1", "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.1.tgz", diff --git a/package.json b/package.json index 635434c2..47e09f55 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,10 @@ "benchmark:capture": "node benchmarks/capture-latency.js", "echo:probe": "node ./scripts/think-echo-capability-probe.mjs", "hexagonal-boundary:ratchet": "node ./scripts/hexagonal-boundary-ratchet.mjs", - "repair:v17-mind": "node ./scripts/repair-v17-mind.mjs", + "generate:memory:wesley": "node ./scripts/generate-think-memory-metadata.mjs", + "generate:memory:sdk": "node ./scripts/render-think-memory-sdk.mjs", + "generate:memory": "npm run generate:memory:wesley && npm run generate:memory:sdk", + "migrate:v19-native": "node ./scripts/convert-v19-mind.mjs", "runtime-truth:ratchet": "node ./scripts/runtime-truth-ratchet.mjs" }, "engines": { @@ -43,7 +46,7 @@ "@flyingrobots/bijou-node": "7.1.0", "@flyingrobots/bijou-tui": "7.1.0", "@git-stunts/alfred": "0.10.3", - "@git-stunts/git-warp": "^18.2.1", + "@git-stunts/git-warp": "^19.0.2", "@git-stunts/plumbing": "3.0.3", "@modelcontextprotocol/sdk": "^1.29.0", "zod": "^4.3.6" diff --git a/scripts/atomic-publish-path.swift b/scripts/atomic-publish-path.swift new file mode 100644 index 00000000..15697409 --- /dev/null +++ b/scripts/atomic-publish-path.swift @@ -0,0 +1,108 @@ +#!/usr/bin/env swift + +import Darwin +import Foundation + +enum AtomicPublishError: Error, CustomStringConvertible { + case usage + case requiresAbsolutePath(String) + case missingDirectory(String) + case symbolicLink(String) + case targetExists(String) + case targetInspectionFailed(String) + case differentFilesystems(String, String) + case publishFailed(String) + + var description: String { + switch self { + case .usage: + return "usage: atomic-publish-path " + case let .requiresAbsolutePath(path): + return "path must be absolute: \(path)" + case let .missingDirectory(path): + return "directory not found: \(path)" + case let .symbolicLink(path): + return "refusing to publish a symbolic link: \(path)" + case let .targetExists(path): + return "refusing to overwrite existing target: \(path)" + case let .targetInspectionFailed(path): + return "could not inspect target path: \(path)" + case let .differentFilesystems(source, target): + return "source and target parent must share a filesystem: \(source), \(target)" + case let .publishFailed(message): + return message + } + } +} + +struct DirectoryIdentity { + let device: dev_t + let path: String +} + +func requireAbsolutePath(_ path: String) throws { + guard path.hasPrefix("/") else { + throw AtomicPublishError.requiresAbsolutePath(path) + } +} + +func inspectDirectory(_ path: String) throws -> DirectoryIdentity { + try requireAbsolutePath(path) + + var metadata = stat() + guard lstat(path, &metadata) == 0 else { + throw AtomicPublishError.missingDirectory(path) + } + + let kind = metadata.st_mode & S_IFMT + guard kind != S_IFLNK else { + throw AtomicPublishError.symbolicLink(path) + } + guard kind == S_IFDIR else { + throw AtomicPublishError.missingDirectory(path) + } + + return DirectoryIdentity(device: metadata.st_dev, path: path) +} + +func requireAbsentTarget(_ path: String) throws { + try requireAbsolutePath(path) + + var metadata = stat() + errno = 0 + if lstat(path, &metadata) == 0 { + throw AtomicPublishError.targetExists(path) + } + guard errno == ENOENT else { + throw AtomicPublishError.targetInspectionFailed(path) + } +} + +func publish(_ source: DirectoryIdentity, targetPath: String) throws { + try requireAbsentTarget(targetPath) + + let targetParentPath = (targetPath as NSString).deletingLastPathComponent + let targetParent = try inspectDirectory(targetParentPath) + guard source.device == targetParent.device else { + throw AtomicPublishError.differentFilesystems(source.path, targetPath) + } + + errno = 0 + guard renamex_np(source.path, targetPath, UInt32(RENAME_EXCL)) == 0 else { + let detail = String(cString: strerror(errno)) + throw AtomicPublishError.publishFailed("atomic directory publish failed: \(detail)") + } +} + +do { + guard CommandLine.arguments.count == 3 else { + throw AtomicPublishError.usage + } + let source = try inspectDirectory(CommandLine.arguments[1]) + let targetPath = CommandLine.arguments[2] + try publish(source, targetPath: targetPath) + print("published \(source.path) -> \(targetPath)") +} catch { + FileHandle.standardError.write(Data("atomic-publish-path: \(error)\n".utf8)) + exit(1) +} diff --git a/scripts/atomic-swap-paths.swift b/scripts/atomic-swap-paths.swift new file mode 100644 index 00000000..19de1922 --- /dev/null +++ b/scripts/atomic-swap-paths.swift @@ -0,0 +1,81 @@ +#!/usr/bin/env swift + +import Darwin +import Foundation + +enum AtomicSwapError: Error, CustomStringConvertible { + case usage + case requiresAbsolutePath(String) + case missingDirectory(String) + case symbolicLink(String) + case differentFilesystems(String, String) + case swapFailed(String) + + var description: String { + switch self { + case .usage: + return "usage: atomic-swap-paths " + case let .requiresAbsolutePath(path): + return "path must be absolute: \(path)" + case let .missingDirectory(path): + return "directory not found: \(path)" + case let .symbolicLink(path): + return "refusing to swap a symbolic link: \(path)" + case let .differentFilesystems(first, second): + return "directories must share a filesystem: \(first), \(second)" + case let .swapFailed(message): + return message + } + } +} + +struct DirectoryIdentity { + let device: dev_t + let path: String +} + +func inspectDirectory(_ path: String) throws -> DirectoryIdentity { + guard path.hasPrefix("/") else { + throw AtomicSwapError.requiresAbsolutePath(path) + } + + var metadata = stat() + guard lstat(path, &metadata) == 0 else { + throw AtomicSwapError.missingDirectory(path) + } + + let kind = metadata.st_mode & S_IFMT + guard kind != S_IFLNK else { + throw AtomicSwapError.symbolicLink(path) + } + guard kind == S_IFDIR else { + throw AtomicSwapError.missingDirectory(path) + } + + return DirectoryIdentity(device: metadata.st_dev, path: path) +} + +func swap(_ first: DirectoryIdentity, _ second: DirectoryIdentity) throws { + guard first.device == second.device else { + throw AtomicSwapError.differentFilesystems(first.path, second.path) + } + + errno = 0 + guard renamex_np(first.path, second.path, UInt32(RENAME_SWAP)) == 0 else { + let detail = String(cString: strerror(errno)) + throw AtomicSwapError.swapFailed("atomic directory swap failed: \(detail)") + } +} + +do { + guard CommandLine.arguments.count == 3 else { + throw AtomicSwapError.usage + } + let first = try inspectDirectory(CommandLine.arguments[1]) + let second = try inspectDirectory(CommandLine.arguments[2]) + try swap(first, second) + print("swapped \(first.path) <-> \(second.path)") +} catch { + FileHandle.standardError.write(Data("atomic-swap-paths: \(error)\n".utf8)) + exit(1) +} diff --git a/scripts/convert-v19-mind-cli.mjs b/scripts/convert-v19-mind-cli.mjs new file mode 100644 index 00000000..233dcafb --- /dev/null +++ b/scripts/convert-v19-mind-cli.mjs @@ -0,0 +1,89 @@ +import { ConvertV19MindError } from './convert-v19-mind-support.mjs'; + +const BOOLEAN_OPTIONS = Object.freeze({ + '--dry-run': 'dryRun', + '--help': 'help', + '--json': 'json', + '-h': 'help', +}); +const VALUE_OPTIONS = Object.freeze({ + '--inventory-in': 'inventoryIn', + '--inventory-out': 'inventoryOut', + '--source': 'source', + '--target': 'target', +}); + +export function usage() { + return [ + 'Usage:', + ' node scripts/convert-v19-mind.mjs --source --inventory-out [--json]', + ' node scripts/convert-v19-mind.mjs --inventory-in --target [--json]', + ' node scripts/convert-v19-mind.mjs --inventory-in --dry-run [--json]', + '', + 'Options:', + ' --source Disposable all-ref copy of the legacy Think mind.', + ' --inventory-out New, checksummed inventory file; never overwritten.', + ' --inventory-in Existing checksummed inventory file to validate/import.', + ' --target Empty Git repository to populate with native v19 data.', + ' --dry-run Validate an inventory without writing a native target.', + ' --json Emit a machine-readable report.', + '', + 'Legacy extraction and native import are deliberately separate invocations.', + 'The rejected generic record is read once into an immutable inventory and is', + 'never admitted to the native target or linked into the Think runtime.', + ].join('\n'); +} + +export function parseConvertArgs(argv) { + const parsed = { + dryRun: false, + help: false, + inventoryIn: null, + inventoryOut: null, + json: false, + source: null, + target: null, + }; + for (let index = 0; index < argv.length;) { + index += consumeConvertArg(argv, index, parsed); + } + return Object.freeze(parsed); +} + +function consumeConvertArg(argv, index, parsed) { + const arg = argv[index]; + const booleanProperty = BOOLEAN_OPTIONS[arg]; + if (booleanProperty) { + parsed[booleanProperty] = true; + return 1; + } + const valueProperty = VALUE_OPTIONS[arg]; + if (!valueProperty) { + throw new ConvertV19MindError(`Unknown argument: ${arg}`, 'convert_v19_mind.usage'); + } + const value = argv[index + 1]; + if (!value || value.startsWith('--')) { + throw new ConvertV19MindError(`${arg} requires a value`, 'convert_v19_mind.usage'); + } + parsed[valueProperty] = value; + return 2; +} + +export function formatReport(report, json) { + return json ? JSON.stringify(report) : JSON.stringify(report, null, 2); +} + +export function formatFailure(error, json) { + const payload = failurePayload(error); + return json + ? JSON.stringify(payload) + : `${payload.code}: ${payload.message}`; +} + +function failurePayload(error) { + return { + code: error?.code ?? 'convert_v19_mind.unexpected', + message: error instanceof Error ? error.message : String(error), + ...(error?.details === undefined ? {} : { details: error.details }), + }; +} diff --git a/scripts/convert-v19-mind-source.mjs b/scripts/convert-v19-mind-source.mjs new file mode 100644 index 00000000..26c06117 --- /dev/null +++ b/scripts/convert-v19-mind-source.mjs @@ -0,0 +1,148 @@ +import { + decodeNativeDocument, + INDEX_DOCUMENT_KEY, + INDEX_PAGE_DOCUMENT_KEY, + MEMORY_DOCUMENT_KEY, +} from '../src/store/native-document.js'; +import { + ConvertV19MindError, + deepFreeze, + isRecord, + stableStringify, +} from './convert-v19-mind-support.mjs'; + +const NATIVE_SOURCE_PROPERTY_KINDS = Object.freeze([ + [MEMORY_DOCUMENT_KEY, 'memory-object'], + [INDEX_DOCUMENT_KEY, 'memory-index'], + [INDEX_PAGE_DOCUMENT_KEY, 'memory-index-page'], +]); + +export function collectNativeSourceProperties(state, modules, liveNodeIds) { + const liveNodeIdSet = new Set(liveNodeIds); + const nativeNodeIds = new Set(); + const documents = []; + const invalidNodeIds = []; + for (const [encodedKey, register] of state.prop) { + const decodedKey = decodeNativeSourceKey(encodedKey, modules); + if (!decodedKey || !liveNodeIdSet.has(decodedKey.nodeId)) { + continue; + } + const decoded = decodeNativeSourceValue(register.value, decodedKey.kind); + if (decoded === null) { + invalidNodeIds.push(decodedKey.nodeId); + continue; + } + nativeNodeIds.add(decodedKey.nodeId); + documents.push(...documentsFromNativeProperty(decodedKey, decoded)); + } + return Object.freeze({ documents, invalidNodeIds, nativeNodeIds }); +} + +function decodeNativeSourceKey(encodedKey, modules) { + if (modules.isEdgePropKey(encodedKey)) { + return null; + } + const { nodeId, propKey } = modules.decodePropKey(encodedKey); + const kind = NATIVE_SOURCE_PROPERTY_KINDS + .find(([key]) => key === propKey)?.[1] ?? null; + return kind === null ? null : Object.freeze({ kind, nodeId }); +} + +function decodeNativeSourceValue(value, kind) { + const bytes = value instanceof Uint8Array + ? value + : value?.toUint8Array?.(); + return decodeNativeDocument(bytes, kind); +} + +function documentsFromNativeProperty({ kind, nodeId }, value) { + if (kind === 'memory-index-page') { + return Array.isArray(value.entries) + ? value.entries.map(requireNativeSourceDocument) + : []; + } + if (kind !== 'memory-object') { + return []; + } + const document = requireNativeSourceDocument(value); + requireNativeDocumentSubject(document, nodeId); + return [document]; +} + +function requireNativeSourceDocument(value) { + if (!isRecord(value) || typeof value.id !== 'string' || value.id.length === 0) { + throw new ConvertV19MindError( + 'Native source contains an invalid document', + 'convert_v19_mind.native_source_invalid' + ); + } + return deepFreeze({ ...value }); +} + +function requireNativeDocumentSubject(document, nodeId) { + if (document.id !== nodeId) { + throw new ConvertV19MindError( + `Native source document ${document.id} does not match subject ${nodeId}`, + 'convert_v19_mind.native_source_invalid' + ); + } +} + +export function requireCompleteSourceRecords(legacy, native, liveNodeIds) { + requireValidLegacyRecords(legacy.invalidRecordIds); + requireValidNativeProperties(native.invalidNodeIds); + const covered = coveredSourceNodeIds(legacy.records, native); + const firstMissing = liveNodeIds.find(id => !covered.has(id)) ?? null; + if (firstMissing !== null) { + throw new ConvertV19MindError( + `Think materialized ${liveNodeIds.length} live nodes but source records do not cover ${firstMissing}`, + 'convert_v19_mind.record_missing' + ); + } +} + +function requireValidLegacyRecords(invalidRecordIds) { + if (invalidRecordIds.length > 0) { + throw new ConvertV19MindError( + `Legacy Think contains ${invalidRecordIds.length} invalid live records; first: ${invalidRecordIds[0]}`, + 'convert_v19_mind.record_invalid' + ); + } +} + +function requireValidNativeProperties(invalidNodeIds) { + if (invalidNodeIds.length > 0) { + throw new ConvertV19MindError( + `Native Think contains ${invalidNodeIds.length} invalid live properties; first: ${invalidNodeIds[0]}`, + 'convert_v19_mind.native_source_invalid' + ); + } +} + +function coveredSourceNodeIds(records, native) { + return new Set([ + ...records.keys(), + ...native.nativeNodeIds, + ...native.documents.map(document => document.id), + ]); +} + +export function mergeNativeSourceDocuments(legacyDocuments, nativeDocuments) { + const documents = new Map( + legacyDocuments.map(document => [document.id, document]) + ); + for (const document of nativeDocuments) { + requireCompatibleDocument(documents.get(document.id), document); + documents.set(document.id, document); + } + return [...documents.values()]; +} + +function requireCompatibleDocument(existing, document) { + if (existing && stableStringify(existing) !== stableStringify(document)) { + throw new ConvertV19MindError( + `Legacy and native source documents conflict for ${document.id}`, + 'convert_v19_mind.source_conflict' + ); + } +} diff --git a/scripts/convert-v19-mind-support.mjs b/scripts/convert-v19-mind-support.mjs new file mode 100644 index 00000000..50318bc9 --- /dev/null +++ b/scripts/convert-v19-mind-support.mjs @@ -0,0 +1,152 @@ +import { createHash } from 'node:crypto'; + +export class ConvertV19MindError extends Error { + constructor(message, code = 'convert_v19_mind.error') { + super(message); + this.name = 'ConvertV19MindError'; + this.code = code; + } +} + +export class InventoryEncodingError extends ConvertV19MindError { + constructor(message) { + super(message, 'convert_v19_mind.snapshot_invalid'); + this.name = 'InventoryEncodingError'; + } +} + +export function summarizeInventory(inventory) { + return Object.freeze({ + documentCount: inventory.documents.length, + edgeCount: inventory.edges.length, + kinds: Object.freeze(Object.fromEntries( + [...inventory.byKind] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([kind, entries]) => [kind, entries.length]) + )), + }); +} + +export function summarizeProjection(sourceInventory, projectedInventory) { + return Object.freeze({ + droppedDocumentCount: + sourceInventory.documents.length - projectedInventory.documents.length, + droppedEdgeCount: sourceInventory.edges.length - projectedInventory.edges.length, + }); +} + +export function requireInventorySummary(value) { + if (!isInventorySummary(value)) { + throw new ConvertV19MindError( + 'Inventory contains an invalid summary', + 'convert_v19_mind.snapshot_invalid' + ); + } + const kinds = {}; + for (const [kind, count] of Object.entries(value.kinds).sort(compareEntries)) { + requireInventoryKindCount(kind, count); + kinds[kind] = count; + } + return deepFreeze({ + documentCount: value.documentCount, + edgeCount: value.edgeCount, + kinds, + }); +} + +function isInventorySummary(value) { + if (!isRecord(value) || !isRecord(value.kinds)) { + return false; + } + return isNonNegativeInteger(value.documentCount) + && isNonNegativeInteger(value.edgeCount); +} + +function requireInventoryKindCount(kind, count) { + if (kind.length === 0 || !isNonNegativeInteger(count)) { + throw new ConvertV19MindError( + 'Inventory contains an invalid kind count', + 'convert_v19_mind.snapshot_invalid' + ); + } +} + +function isNonNegativeInteger(value) { + return Number.isInteger(value) && value >= 0; +} + +function compareEntries([left], [right]) { + return left.localeCompare(right); +} + +export function sampleDocuments(documents) { + if (documents.length <= 3) { + return documents; + } + return [ + documents[0], + documents[Math.floor(documents.length / 2)], + documents.at(-1), + ]; +} + +export function chunk(values, size) { + const chunks = []; + for (let index = 0; index < values.length; index += size) { + chunks.push(Object.freeze(values.slice(index, index + size))); + } + return chunks; +} + +export function compareDocumentsOldestFirst(left, right) { + const leftSort = typeof left.sortKey === 'string' ? left.sortKey : left.id; + const rightSort = typeof right.sortKey === 'string' ? right.sortKey : right.id; + return leftSort.localeCompare(rightSort) || left.id.localeCompare(right.id); +} + +export function compareEdges(left, right) { + return left.from.localeCompare(right.from) + || left.label.localeCompare(right.label) + || left.to.localeCompare(right.to); +} + +export function isRecord(value) { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function isSha256(value) { + return typeof value === 'string' && /^[a-f0-9]{64}$/u.test(value); +} + +export function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +export function stableStringify(value) { + if (Array.isArray(value)) { + return `[${value.map(stableStringify).join(',')}]`; + } + if (isRecord(value)) { + return `{${Object.keys(value) + .sort() + .map(key => `${JSON.stringify(key)}:${stableStringify(value[key])}`) + .join(',')}}`; + } + const encoded = JSON.stringify(value); + if (encoded === undefined) { + throw new InventoryEncodingError( + 'Inventory contains a value that cannot be encoded' + ); + } + return encoded; +} + +export function deepFreeze(value) { + if (!isRecord(value) && !Array.isArray(value)) { + return value; + } + for (const child of Object.values(value)) { + deepFreeze(child); + } + return Object.freeze(value); +} diff --git a/scripts/convert-v19-mind.mjs b/scripts/convert-v19-mind.mjs new file mode 100644 index 00000000..bcbad8ff --- /dev/null +++ b/scripts/convert-v19-mind.mjs @@ -0,0 +1,984 @@ +#!/usr/bin/env node + +import { existsSync } from 'node:fs'; +import { readFile, writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { spawnSync } from 'node:child_process'; + +import { GIT_BINARY, THINK_GIT_CONFIG_ARGS } from '../src/git.js'; +import { + ARTIFACT_PREFIX, + CLASSIFICATION_PREFIX, + GRAPH_META_ID, + GRAPH_NAME, + KEYWORD_PREFIX, + PIPELINE_RUN_PREFIX, + READ_MODEL_PREFIX, + TOPIC_PREFIX, +} from '../src/store/constants.js'; +import { + decodeNativeDocument, + encodeNativeDocument, +} from '../src/store/native-document.js'; +import { + NATIVE_INDEX_PAGE_SIZE, + readIndexedMemoryDocument, +} from '../src/store/native-index.js'; +import { + closeNativeMemory, + openNativeMemory, +} from '../src/store/native-runtime.js'; +import { + formatFailure, + formatReport, + parseConvertArgs, + usage, +} from './convert-v19-mind-cli.mjs'; +import { + collectNativeSourceProperties, + mergeNativeSourceDocuments, + requireCompleteSourceRecords, +} from './convert-v19-mind-source.mjs'; +import { + chunk, + compareDocumentsOldestFirst, + compareEdges, + ConvertV19MindError, + deepFreeze, + isRecord, + isSha256, + requireInventorySummary, + sampleDocuments, + sha256, + stableStringify, + summarizeInventory, + summarizeProjection, +} from './convert-v19-mind-support.mjs'; + +const LEGACY_RECORD_KEY = 'think.record.v1'; +const LEGACY_RECORD_VERSION = 1; +const LEGACY_CATALOG_COUNT = 4; +const LEGACY_CATALOG_PREFIX = `${READ_MODEL_PREFIX}v19:catalog:`; +const NATIVE_INDEX_PREFIX = `${READ_MODEL_PREFIX}v19:index:`; +const CONVERTER_WRITER = 'think-native-v19-converter'; +const EVACUATION_MATERIALIZATION_NAMESPACE = 'think-v19-migration'; +const INVENTORY_FORMAT = 'think.native-v19.inventory'; +const INVENTORY_VERSION = 1; +const IMPORT_CONCURRENCY = 8; +const READ_CONCURRENCY = 16; +const RECOMPUTABLE_PREFIXES = Object.freeze([ + ARTIFACT_PREFIX, + CLASSIFICATION_PREFIX, + KEYWORD_PREFIX, + PIPELINE_RUN_PREFIX, + READ_MODEL_PREFIX, + TOPIC_PREFIX, +]); +const moduleRequire = createRequire(import.meta.url); +const gitWarpPackageRoot = path.dirname( + moduleRequire.resolve('@git-stunts/git-warp/package.json') +); +let gitWarpEvacuationModulesPromise = null; + +export { ConvertV19MindError, parseConvertArgs, usage }; + +export async function convertV19Mind(options) { + const mode = requireConversionMode(options); + if (mode === 'extract') { + return await extractLegacyInventory(options); + } + return await convertInventory(options); +} + +async function extractLegacyInventory(options) { + const sourceDir = requireRepository(options.source, '--source'); + const outputPath = path.resolve(options.inventoryOut); + const refsBefore = repositoryRefsSha256(sourceDir); + const extractedInventory = await readLegacyInventory(sourceDir); + requireStableSourceRefs(sourceDir, refsBefore); + const outputSnapshot = createInventorySnapshot({ + inventory: extractedInventory, + sourceDir, + sourceRefsSha256: refsBefore, + }); + await writeInventorySnapshot(outputPath, outputSnapshot); + const persisted = await loadInventorySnapshot(outputPath); + return Object.freeze({ + status: 'inventoried', + sourceDir, + inventoryPath: outputPath, + sourceRefsSha256: refsBefore, + manifestSha256: persisted.snapshot.manifestSha256, + ...persisted.snapshot.summary, + verified: true, + }); +} + +function requireStableSourceRefs(sourceDir, refsBefore) { + if (refsBefore !== repositoryRefsSha256(sourceDir)) { + throw new ConvertV19MindError( + 'Legacy source refs changed while the inventory was being read', + 'convert_v19_mind.source_changed' + ); + } +} + +async function convertInventory(options) { + const inputPath = path.resolve(options.inventoryIn); + const loaded = await loadInventorySnapshot(inputPath); + const inventory = projectNativeInventory(loaded.inventory); + const context = Object.freeze({ inputPath, inventory, loaded }); + if (options.dryRun) { + return inventoryVerificationReport(context); + } + const targetDir = requireRepository(options.target, '--target'); + return await importNativeInventory(context, targetDir); +} + +function inventoryVerificationReport({ inputPath, inventory, loaded }) { + return Object.freeze({ + status: 'inventory-verified', + ...inventoryReportBase(inputPath, inventory, loaded), + verified: true, + }); +} + +async function importNativeInventory(context, targetDir) { + requireEmptyTarget(targetDir); + await runPartitionedWrites( + targetDir, + 'index', + [...context.inventory.byKind], + async (workerMemory, [kind, documents]) => { + await writeNativeIndex(workerMemory, kind, documents); + } + ); + await repairConvertedTarget(targetDir); + const verification = await verifyConversion(targetDir, context.inventory); + if (!verification.verified) { + const error = new ConvertV19MindError( + 'Native v19 conversion verification failed', + 'convert_v19_mind.verification_failed' + ); + error.details = verification; + throw error; + } + return Object.freeze({ + status: 'converted', + ...inventoryReportBase( + context.inputPath, + context.inventory, + context.loaded + ), + targetDir, + verified: true, + verification, + }); +} + +function inventoryReportBase(inputPath, inventory, loaded) { + return { + inventoryPath: inputPath, + sourceDir: loaded.snapshot.source.repoDir, + sourceRefsSha256: loaded.snapshot.source.refsSha256, + manifestSha256: loaded.snapshot.manifestSha256, + sourceSummary: loaded.snapshot.summary, + ...summarizeInventory(inventory), + ...summarizeProjection(loaded.inventory, inventory), + }; +} + +async function repairConvertedTarget(targetDir) { + const memory = await openNativeMemory(targetDir, { + writerId: CONVERTER_WRITER, + }); + try { + await memory.repairBasis(); + } finally { + await closeNativeMemory(targetDir); + } +} + +export function buildNativeInventory(records) { + const documents = []; + const edges = []; + for (const [id, record] of records) { + const document = Object.freeze({ + id, + ...record.props, + ...(record.text === null ? {} : { text: record.text }), + }); + documents.push(document); + for (const edge of record.outgoing) { + edges.push(Object.freeze({ + from: id, + to: edge.id, + label: edge.label, + })); + } + } + return buildInventoryFromNativeDocuments(documents, edges); +} + +export function createInventorySnapshot({ + inventory, + sourceDir, + sourceRefsSha256, +}) { + if (!isSha256(sourceRefsSha256)) { + throw new ConvertV19MindError( + 'Source ref digest must be a SHA-256 value', + 'convert_v19_mind.snapshot_invalid' + ); + } + const payload = Object.freeze({ + format: INVENTORY_FORMAT, + version: INVENTORY_VERSION, + source: Object.freeze({ + repoDir: path.resolve(sourceDir), + refsSha256: sourceRefsSha256, + }), + summary: summarizeInventory(inventory), + documents: inventory.documents, + edges: inventory.edges, + }); + return deepFreeze({ + ...payload, + manifestSha256: sha256(stableStringify(payload)), + }); +} + +export function projectNativeInventory(sourceInventory) { + const capturedThoughtIds = new Set( + sourceInventory.documents + .filter(document => document.kind === 'capture') + .map(document => document.thoughtId) + .filter(value => typeof value === 'string' && value.length > 0) + ); + const documents = sourceInventory.documents.filter(document => ( + !isRecomputableDocument(document, capturedThoughtIds) + )); + return buildInventoryFromNativeDocuments([...documents], []); +} + +export function parseInventorySnapshot(text) { + const parsed = parseInventoryJson(text); + requireInventoryEnvelope(parsed); + const documents = requireUniqueNativeDocuments(parsed.documents); + const edges = parsed.edges.map(requireNativeEdge); + const inventory = buildInventoryFromNativeDocuments(documents, edges); + const summary = requireInventorySummary(parsed.summary); + requireMatchingInventorySummary(summary, inventory); + const payload = createParsedInventoryPayload(parsed, summary, documents, edges); + const expectedSha256 = sha256(stableStringify(payload)); + requireMatchingManifest(parsed.manifestSha256, expectedSha256); + return Object.freeze({ + inventory, + snapshot: deepFreeze({ + ...payload, + manifestSha256: expectedSha256, + }), + }); +} + +function parseInventoryJson(text) { + try { + return JSON.parse(text); + } catch { + throw new ConvertV19MindError( + 'Inventory is not valid JSON', + 'convert_v19_mind.snapshot_invalid' + ); + } +} + +function requireInventoryEnvelope(value) { + if (!isRecord(value)) { + throw invalidInventoryStructureError(); + } + requireInventoryIdentity(value); + requireInventorySource(value.source); + if (!isRecord(value.summary)) { + throw invalidInventoryStructureError(); + } + if (!Array.isArray(value.documents) || !Array.isArray(value.edges)) { + throw invalidInventoryStructureError(); + } +} + +function requireInventoryIdentity(value) { + if (value.format !== INVENTORY_FORMAT || value.version !== INVENTORY_VERSION) { + throw invalidInventoryStructureError(); + } + if (!isSha256(value.manifestSha256)) { + throw invalidInventoryStructureError(); + } +} + +function requireInventorySource(source) { + if (!isRecord(source) || !isSha256(source.refsSha256)) { + throw invalidInventoryStructureError(); + } + if (typeof source.repoDir !== 'string' || source.repoDir.length === 0) { + throw invalidInventoryStructureError(); + } +} + +function invalidInventoryStructureError() { + return new ConvertV19MindError( + 'Inventory structure is invalid', + 'convert_v19_mind.snapshot_invalid' + ); +} + +function requireUniqueNativeDocuments(values) { + const documents = values.map(requireNativeDocument); + const ids = new Set(); + for (const document of documents) { + if (ids.has(document.id)) { + throw new ConvertV19MindError( + `Inventory contains duplicate document id: ${document.id}`, + 'convert_v19_mind.snapshot_invalid' + ); + } + ids.add(document.id); + } + return documents; +} + +function requireMatchingInventorySummary(summary, inventory) { + if (stableStringify(summary) !== stableStringify(summarizeInventory(inventory))) { + throw new ConvertV19MindError( + 'Inventory summary does not match its documents and edges', + 'convert_v19_mind.snapshot_invalid' + ); + } +} + +function createParsedInventoryPayload(parsed, summary, documents, edges) { + return deepFreeze({ + format: INVENTORY_FORMAT, + version: INVENTORY_VERSION, + source: { + repoDir: parsed.source.repoDir, + refsSha256: parsed.source.refsSha256, + }, + summary, + documents, + edges, + }); +} + +function requireMatchingManifest(actual, expected) { + if (actual !== expected) { + throw new ConvertV19MindError( + 'Inventory checksum does not match its manifest', + 'convert_v19_mind.snapshot_checksum_mismatch' + ); + } +} + +function buildInventoryFromNativeDocuments(documents, edges) { + const byKind = new Map(); + for (const document of documents) { + const kind = typeof document.kind === 'string' && document.kind.length > 0 + ? document.kind + : null; + if (kind) { + const entries = byKind.get(kind) ?? []; + entries.push(document); + byKind.set(kind, entries); + } + } + for (const entries of byKind.values()) { + entries.sort(compareDocumentsOldestFirst); + Object.freeze(entries); + } + documents.sort((left, right) => left.id.localeCompare(right.id)); + edges.sort(compareEdges); + return Object.freeze({ + byKind, + documents: Object.freeze(documents), + edges: Object.freeze(edges), + }); +} + +async function loadInventorySnapshot(inventoryPath) { + let contents; + try { + contents = await readFile(inventoryPath, 'utf8'); + } catch (error) { + throw new ConvertV19MindError( + `Unable to read inventory ${inventoryPath}: ${error.message}`, + 'convert_v19_mind.snapshot_read_failed' + ); + } + return parseInventorySnapshot(contents); +} + +async function writeInventorySnapshot(inventoryPath, snapshot) { + try { + await writeFile( + inventoryPath, + `${JSON.stringify(snapshot, null, 2)}\n`, + { encoding: 'utf8', flag: 'wx' } + ); + } catch (error) { + const code = error?.code === 'EEXIST' + ? 'convert_v19_mind.snapshot_exists' + : 'convert_v19_mind.snapshot_write_failed'; + throw new ConvertV19MindError( + `Unable to create inventory ${inventoryPath}: ${error.message}`, + code + ); + } +} + +async function readLegacyInventory(repoDir) { + const modules = await loadGitWarpEvacuationModules(); + const state = await materializeLegacyState(repoDir, modules); + const liveNodeIds = [...state.nodeAlive.elements()].sort(); + const collected = collectLegacyRecords(state, modules, liveNodeIds); + const native = collectNativeSourceProperties(state, modules, liveNodeIds); + requireCompleteSourceRecords(collected, native, liveNodeIds); + requireLegacyCatalogs(collected.records); + return mergeSourceInventories( + buildNativeInventory(collected.records), + native.documents + ); +} + +function collectLegacyRecords(state, modules, liveNodeIds) { + const liveNodeIdSet = new Set(liveNodeIds); + const records = new Map(); + const invalidRecordIds = []; + for (const [encodedKey, register] of state.prop) { + if (modules.isEdgePropKey(encodedKey)) { + continue; + } + const { nodeId, propKey } = modules.decodePropKey(encodedKey); + if (propKey !== LEGACY_RECORD_KEY || !liveNodeIdSet.has(nodeId)) { + continue; + } + const record = parseLegacyRecord(register.value); + if (record === null) { + invalidRecordIds.push(nodeId); + continue; + } + records.set(nodeId, record); + } + return Object.freeze({ invalidRecordIds, records }); +} + +export function mergeSourceInventories(legacyInventory, nativeDocuments) { + return buildInventoryFromNativeDocuments( + mergeNativeSourceDocuments(legacyInventory.documents, nativeDocuments), + [...legacyInventory.edges] + ); +} + +function requireLegacyCatalogs(records) { + const missingCatalogIds = legacyCatalogIds().filter(id => !records.has(id)); + if (missingCatalogIds.length > 0) { + throw new ConvertV19MindError( + `Legacy Think catalog records are missing: ${missingCatalogIds.join(', ')}`, + 'convert_v19_mind.legacy_catalog_missing' + ); + } +} + +async function materializeLegacyState(repoDir, modules) { + const storage = await modules.GitStorage.open({ cwd: repoDir }); + const binding = modules.resolveWarpStorage(storage); + const ports = modules.getDefaultRuntimeHostNodePorts(); + const runtimeStorage = migrationRuntimeStorage(binding.runtimeStorage, modules); + let graph = null; + try { + graph = await modules.openRuntimeHostProduct({ + persistence: binding.history, + runtimeStorage, + graphName: GRAPH_NAME, + writerId: CONVERTER_WRITER, + codec: new modules.V18CheckpointMigrationCodec(), + crypto: ports.crypto, + trustCrypto: ports.trustCrypto, + commitMessageCodec: ports.commitMessageCodec, + stateCache: null, + autoMaterialize: false, + }); + return await graph.materialize(); + } finally { + if (graph !== null) { + await graph.close(); + } + await storage.close(); + } +} + +function migrationRuntimeStorage(runtimeStorage, modules) { + return Object.freeze({ + async createRuntimeStorageServices(options) { + const services = await runtimeStorage.createRuntimeStorageServices(options); + const { materializations } = services; + const withoutTrie = { ...services }; + Reflect.deleteProperty(withoutTrie, 'materializations'); + Reflect.deleteProperty(withoutTrie, 'trie'); + return Object.freeze({ + ...withoutTrie, + materializations: transientMaterializationStore(materializations, modules), + }); + }, + }); +} + +function transientMaterializationStore(underlying, modules) { + return Object.freeze({ + openWorkspace: rejectMaterializationWorkspace, + retain: request => retainTransientMaterialization(request, modules), + acquireExact: resolveNull, + acquireBestCompatiblePredecessor: resolveNull, + loadReplayBasis: resolveNull, + close: () => underlying.close(), + }); +} + +function rejectMaterializationWorkspace() { + return Promise.reject(new ConvertV19MindError( + 'Legacy evacuation cannot open a materialization workspace', + 'convert_v19_mind.materialization_write_rejected' + )); +} + +function retainTransientMaterialization(request, modules) { + const bundle = new modules.BundleHandle(`transient:${request.stateHash}`); + return Promise.resolve(new modules.MaterializationHandle({ + laneName: GRAPH_NAME, + bundle, + coordinate: request.coordinate, + roots: request.roots, + stateHash: request.stateHash, + retention: new modules.StorageRetentionWitness({ + handle: bundle, + policy: 'evictable', + reachability: 'volatile', + root: transientRetentionRoot(modules), + observedAt: new Date().toISOString(), + }), + })); +} + +function transientRetentionRoot(modules) { + return new modules.StorageRetentionRoot({ + kind: 'expiring-set', + namespace: EVACUATION_MATERIALIZATION_NAMESPACE, + locator: 'memory', + generation: '1', + path: GRAPH_NAME, + }); +} + +function resolveNull() { + return Promise.resolve(null); +} + +async function loadGitWarpEvacuationModules() { + gitWarpEvacuationModulesPromise ??= loadEvacuationModuleSet(); + return await gitWarpEvacuationModulesPromise; +} + +async function loadEvacuationModuleSet() { + const modules = await Promise.all([ + importGitWarpModule('dist/src/application/GitStorage.js'), + importGitWarpModule('dist/src/application/WarpStorageRegistry.js'), + importGitWarpModule('dist/src/application/RuntimeHostNodeDefaults.js'), + importGitWarpModule('dist/src/domain/warp/RuntimeHostProduct.js'), + importGitWarpModule('dist/src/domain/materialization/MaterializationHandle.js'), + importGitWarpModule('dist/src/domain/storage/BundleHandle.js'), + importGitWarpModule('dist/src/domain/storage/StorageRetentionWitness.js'), + importGitWarpModule('dist/src/domain/services/KeyCodec.js'), + importGitWarpModule( + 'dist/scripts/v18-to-v19/V18CheckpointMigrationCodec.js' + ), + ]); + return assembleEvacuationModules(modules); +} + +function assembleEvacuationModules([ + gitStorage, + storageRegistry, + nodeDefaults, + runtimeHostProduct, + materializationHandle, + bundleHandle, + storageRetention, + keyCodec, + checkpointCodec, + ]) { + return Object.freeze({ + GitStorage: gitStorage.default, + resolveWarpStorage: storageRegistry.resolveWarpStorage, + getDefaultRuntimeHostNodePorts: nodeDefaults.getDefaultRuntimeHostNodePorts, + openRuntimeHostProduct: runtimeHostProduct.openRuntimeHostProduct, + MaterializationHandle: materializationHandle.default, + BundleHandle: bundleHandle.default, + StorageRetentionWitness: storageRetention.default, + StorageRetentionRoot: storageRetention.StorageRetentionRoot, + decodePropKey: keyCodec.decodePropKey, + isEdgePropKey: keyCodec.isEdgePropKey, + V18CheckpointMigrationCodec: checkpointCodec.V18CheckpointMigrationCodec, + }); +} + +async function importGitWarpModule(relativePath) { + return await import(pathToFileURL(path.join(gitWarpPackageRoot, relativePath)).href); +} + +function parseLegacyRecord(value) { + if (typeof value !== 'string' || value.length === 0) { + return null; + } + try { + const parsed = JSON.parse(value); + if (!isLegacyRecord(parsed)) { + return null; + } + return Object.freeze({ + props: Object.freeze({ ...parsed.props }), + text: parsed.text, + incoming: Object.freeze(parsed.incoming.map(requireLegacyEdge)), + outgoing: Object.freeze(parsed.outgoing.map(requireLegacyEdge)), + }); + } catch { + return null; + } +} + +function isLegacyRecord(value) { + if (!isRecord(value) || value.version !== LEGACY_RECORD_VERSION) { + return false; + } + if (!isRecord(value.props) || !Array.isArray(value.incoming)) { + return false; + } + if (!Array.isArray(value.outgoing)) { + return false; + } + return value.text === null || typeof value.text === 'string'; +} + +async function writeNativeIndex(memory, kind, documents) { + const pages = chunk(documents, NATIVE_INDEX_PAGE_SIZE); + for (let pageNumber = 0; pageNumber < pages.length; pageNumber += 1) { + const page = Object.freeze({ + kind, + pageNumber, + entries: Object.freeze(pages[pageNumber]), + }); + // eslint-disable-next-line no-await-in-loop -- pages are independently bounded v19 properties + await memory.writeMemoryIndexPage({ + id: nativeIndexPageId(kind, pageNumber), + bytes: encodeNativeDocument('memory-index-page', page), + }); + } + const latest = documents.at(-1) ?? null; + const state = Object.freeze({ + id: nativeIndexId(kind), + kind, + total: documents.length, + latestId: latest?.id ?? null, + headPage: Math.max(0, pages.length - 1), + }); + await memory.writeMemoryIndex({ + id: state.id, + bytes: encodeNativeDocument('memory-index', state), + }); +} + +async function verifyConversion(repoDir, inventory) { + const memory = await openNativeMemory(repoDir, { writerId: CONVERTER_WRITER }); + try { + const expectedCaptures = inventory.byKind.get('capture') ?? []; + const captureState = decodeNativeDocument( + await memory.memoryIndex(nativeIndexId('capture')), + 'memory-index' + ); + const samples = sampleDocuments(inventory.documents); + const sampleResults = await mapConcurrent(samples, async (expected) => { + const actual = await readIndexedMemoryDocument(repoDir, expected.id, { + kinds: [expected.kind], + memory, + }); + return Object.freeze({ + id: expected.id, + matched: JSON.stringify(actual) === JSON.stringify(expected), + }); + }, READ_CONCURRENCY); + const verified = captureState?.total === expectedCaptures.length + && sampleResults.every(result => result.matched); + return Object.freeze({ + verified, + captureCount: captureState?.total ?? null, + expectedCaptureCount: expectedCaptures.length, + samples: Object.freeze(sampleResults), + }); + } finally { + await closeNativeMemory(repoDir); + } +} + +async function mapConcurrent(values, task, concurrency) { + const results = new Array(values.length); + let nextIndex = 0; + async function worker() { + while (nextIndex < values.length) { + const index = nextIndex; + nextIndex += 1; + // eslint-disable-next-line no-await-in-loop -- bounded worker pool + results[index] = await task(values[index], index); + } + } + await Promise.all( + Array.from( + { length: Math.min(concurrency, values.length) }, + () => worker() + ) + ); + return results; +} + +async function runPartitionedWrites(repoDir, phase, values, task) { + if (values.length === 0) { + return; + } + let nextIndex = 0; + const workerCount = Math.min(IMPORT_CONCURRENCY, values.length); + const workers = Array.from({ length: workerCount }, (_unused, workerIndex) => ( + async () => { + const writerId = `${CONVERTER_WRITER}.${phase}.${String(workerIndex).padStart(2, '0')}`; + const memory = await openNativeMemory(repoDir, { writerId }); + while (nextIndex < values.length) { + const index = nextIndex; + nextIndex += 1; + try { + // eslint-disable-next-line no-await-in-loop -- each worker owns one ordered writer head + await task(memory, values[index], index); + } catch (error) { + return error; + } + } + return null; + } + )); + let results; + try { + results = await Promise.all(workers.map(worker => worker())); + } finally { + await closeNativeMemory(repoDir); + } + const failure = results.find(result => result !== null) ?? null; + if (failure !== null) { + throw failure; + } +} + +function legacyCatalogIds() { + return Array.from( + { length: LEGACY_CATALOG_COUNT }, + (_unused, index) => `${LEGACY_CATALOG_PREFIX}${String(index).padStart(2, '0')}` + ); +} + +function nativeIndexId(kind) { + return `${NATIVE_INDEX_PREFIX}${kind}`; +} + +function nativeIndexPageId(kind, number) { + return `${NATIVE_INDEX_PREFIX}${kind}:page:${String(number).padStart(8, '0')}`; +} + +function requireRepository(value, option) { + if (!value) { + throw new ConvertV19MindError(`${option} is required`, 'convert_v19_mind.usage'); + } + const repoDir = path.resolve(value); + if (!existsSync(path.join(repoDir, '.git'))) { + throw new ConvertV19MindError( + `Think repository not found: ${repoDir}`, + 'convert_v19_mind.repo_not_found' + ); + } + return repoDir; +} + +function requireConversionMode(options) { + const hasSource = hasOptionValue(options.source); + const hasInventoryIn = hasOptionValue(options.inventoryIn); + if (hasSource === hasInventoryIn) { + throw new ConvertV19MindError( + 'Exactly one of --source or --inventory-in is required', + 'convert_v19_mind.usage' + ); + } + return hasSource + ? requireExtractMode(options) + : requireInventoryMode(options); +} + +function requireExtractMode(options) { + const invalid = !hasOptionValue(options.inventoryOut) + || hasOptionValue(options.target) + || options.dryRun; + if (invalid) { + throw new ConvertV19MindError( + '--source requires --inventory-out and cannot be combined with --target or --dry-run', + 'convert_v19_mind.usage' + ); + } + return 'extract'; +} + +function requireInventoryMode(options) { + if (hasOptionValue(options.inventoryOut)) { + throw new ConvertV19MindError( + '--inventory-out can only be combined with --source', + 'convert_v19_mind.usage' + ); + } + const hasTarget = hasOptionValue(options.target); + if (options.dryRun === hasTarget) { + throw new ConvertV19MindError( + '--inventory-in requires exactly one of --target or --dry-run', + 'convert_v19_mind.usage' + ); + } + return options.dryRun ? 'validate' : 'import'; +} + +function hasOptionValue(value) { + return typeof value === 'string' && value.length > 0; +} + +function repositoryRefsSha256(repoDir) { + const result = spawnSync( + GIT_BINARY, + [ + ...THINK_GIT_CONFIG_ARGS, + '-C', + repoDir, + 'for-each-ref', + '--format=%(refname) %(objectname)', + ], + { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 } + ); + if (result.status !== 0) { + throw new ConvertV19MindError( + `Unable to snapshot refs in ${repoDir}`, + 'convert_v19_mind.source_inspection_failed' + ); + } + const refs = result.stdout + .split(/\r?\n/u) + .filter(line => line.length > 0) + .sort(); + return sha256(refs.length > 0 ? `${refs.join('\n')}\n` : ''); +} + +function requireEmptyTarget(repoDir) { + const result = spawnSync( + GIT_BINARY, + [ + ...THINK_GIT_CONFIG_ARGS, + '-C', + repoDir, + 'for-each-ref', + '--count=1', + '--format=%(refname)', + ], + { encoding: 'utf8' } + ); + if (result.status !== 0) { + throw new ConvertV19MindError( + `Unable to inspect native target: ${repoDir}`, + 'convert_v19_mind.target_inspection_failed' + ); + } + if (result.stdout.trim().length > 0) { + throw new ConvertV19MindError( + `Native target must contain no refs: ${repoDir}`, + 'convert_v19_mind.target_not_empty' + ); + } +} + +function requireLegacyEdge(value) { + if (!isRecord(value) || typeof value.id !== 'string' || typeof value.label !== 'string') { + throw new ConvertV19MindError( + 'Legacy Think record contains an invalid edge', + 'convert_v19_mind.edge_invalid' + ); + } + return Object.freeze({ id: value.id, label: value.label }); +} + +function requireNativeDocument(value) { + if (!isRecord(value) || typeof value.id !== 'string' || value.id.length === 0) { + throw new ConvertV19MindError( + 'Inventory contains an invalid native document', + 'convert_v19_mind.snapshot_invalid' + ); + } + return deepFreeze({ ...value }); +} + +function requireNativeEdge(value) { + if ( + !isRecord(value) + || typeof value.from !== 'string' + || value.from.length === 0 + || typeof value.to !== 'string' + || value.to.length === 0 + || typeof value.label !== 'string' + || value.label.length === 0 + ) { + throw new ConvertV19MindError( + 'Inventory contains an invalid native edge', + 'convert_v19_mind.snapshot_invalid' + ); + } + return Object.freeze({ + from: value.from, + to: value.to, + label: value.label, + }); +} + +function isRecomputableDocument(document, capturedThoughtIds) { + if (document.id === GRAPH_META_ID) { + return true; + } + if (RECOMPUTABLE_PREFIXES.some(prefix => document.id.startsWith(prefix))) { + return true; + } + return document.kind === 'thought' && capturedThoughtIds.has(document.id); +} + +async function main() { + const args = parseConvertArgs(process.argv.slice(2)); + if (args.help) { + process.stdout.write(`${usage()}\n`); + return; + } + try { + const report = await convertV19Mind(args); + process.stdout.write(`${formatReport(report, args.json)}\n`); + } catch (error) { + process.stderr.write(`${formatFailure(error, args.json)}\n`); + process.exitCode = 1; + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + await main(); +} diff --git a/scripts/generate-think-memory-metadata.mjs b/scripts/generate-think-memory-metadata.mjs new file mode 100644 index 00000000..fa2b5afc --- /dev/null +++ b/scripts/generate-think-memory-metadata.mjs @@ -0,0 +1,28 @@ +import { execFile } from 'node:child_process'; +import path from 'node:path'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); +const REQUIRED_WESLEY_VERSION = '0.3.0-alpha.1'; +const wesley = process.env.WESLEY_BIN || 'wesley'; +const repoRoot = path.resolve(import.meta.dirname, '..'); +const schema = path.join(repoRoot, 'contracts', 'think-git-warp-v19.graphql'); +const output = path.join(repoRoot, 'src', 'generated', 'think-memory.wesley.generated.ts'); + +class ThinkMemoryMetadataGenerationError extends Error {} + +const { stdout } = await execFileAsync(wesley, ['--version']); +if (stdout.trim() !== REQUIRED_WESLEY_VERSION) { + throw new ThinkMemoryMetadataGenerationError( + `Think SDK generation requires Wesley ${REQUIRED_WESLEY_VERSION}; received ${stdout.trim() || 'no version'}` + ); +} + +await execFileAsync(wesley, [ + 'emit', + 'typescript', + '--schema', + schema, + '--out', + output, +]); diff --git a/scripts/hexagonal-boundary-ratchet.mjs b/scripts/hexagonal-boundary-ratchet.mjs index d287b8a2..b4f36d0a 100644 --- a/scripts/hexagonal-boundary-ratchet.mjs +++ b/scripts/hexagonal-boundary-ratchet.mjs @@ -9,14 +9,18 @@ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..' const baselinePath = path.join(repoRoot, 'docs', 'audit', 'hexagonal-boundary-ratchet-baseline.json'); const sourcePrefixes = Object.freeze(['src/', 'bin/', 'scripts/']); const allowedBoundaryFiles = Object.freeze([ + 'scripts/convert-v19-mind.mjs', 'scripts/hexagonal-boundary-ratchet.mjs', - 'scripts/repair-v17-mind.mjs', + 'scripts/render-think-memory-sdk.mjs', 'src/browse-benchmark.js', 'src/browse/adapters/git-warp-worker.js', 'src/browse/adapters/git-warp.js', 'src/cli/commands/doctor.js', 'src/doctor.js', + 'src/generated/think-memory.generated.js', 'src/history/git-warp-read.js', + 'src/store/native-runtime.js', + 'src/store/runtime.js', ]); const substrateTerms = Object.freeze([ { diff --git a/scripts/render-think-memory-sdk.mjs b/scripts/render-think-memory-sdk.mjs new file mode 100644 index 00000000..5fb2c771 --- /dev/null +++ b/scripts/render-think-memory-sdk.mjs @@ -0,0 +1,266 @@ +import { writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +import { + mutationDeclareMemoryObjectOperation, + mutationStoreMemoryDocumentOperation, + mutationStoreMemoryIndexOperation, + mutationStoreMemoryIndexPageOperation, + queryMemoryDocumentOperation, + queryMemoryIndexOperation, + queryMemoryIndexPageOperation, + queryMemoryObjectExistsOperation, +} from '../src/generated/think-memory.wesley.generated.ts'; + +class ThinkMemorySdkGenerationError extends Error {} + +const contracts = Object.freeze([ + [ + mutationDeclareMemoryObjectOperation, + 'MUTATION:declareMemoryObject:NODE_ADD:subject', + [ + 'operationType', + 'fieldName', + 'directives.intent.kind', + 'directives.intent.subject', + ], + ], + [ + mutationStoreMemoryDocumentOperation, + 'MUTATION:storeMemoryDocument:PROPERTY_SET_BYTES:subject:think.memory-document.v1:value', + intentPropertyFields(), + ], + [ + mutationStoreMemoryIndexOperation, + 'MUTATION:storeMemoryIndex:PROPERTY_SET_BYTES:subject:think.memory-index.v1:value', + intentPropertyFields(), + ], + [ + mutationStoreMemoryIndexPageOperation, + 'MUTATION:storeMemoryIndexPage:PROPERTY_SET_BYTES:subject:think.memory-index-page.v1:value', + intentPropertyFields(), + ], + [ + queryMemoryDocumentOperation, + 'QUERY:memoryDocument:think.memory.document:PROPERTY:subject:think.memory-document.v1:EXACTLY_ONE:BYTES', + observerPropertyFields(), + ], + [ + queryMemoryIndexOperation, + 'QUERY:memoryIndex:think.memory.index:PROPERTY:subject:think.memory-index.v1:EXACTLY_ONE:BYTES', + observerPropertyFields(), + ], + [ + queryMemoryIndexPageOperation, + 'QUERY:memoryIndexPage:think.memory.index-page:PROPERTY:subject:think.memory-index-page.v1:EXACTLY_ONE:BYTES', + observerPropertyFields(), + ], + [ + queryMemoryObjectExistsOperation, + 'QUERY:memoryObjectExists:think.memory.object-exists:NODE_EXISTS:subject:EXACTLY_ONE:BOOLEAN', + [ + 'operationType', + 'fieldName', + 'directives.observer.id', + 'directives.observer.reading', + 'directives.observer.subject', + 'directives.observer.cardinality', + 'directives.observer.decoder', + ], + ], +]); + +for (const [operation, expected, fields] of contracts) { + const actual = fields.map(field => readPath(operation, field)).join(':'); + if (actual !== expected) { + throw new ThinkMemorySdkGenerationError( + `Think memory contract drifted at ${operation.fieldName}: expected ${expected}; received ${actual}` + ); + } +} + +const output = process.argv[2] + ? path.resolve(process.argv[2]) + : path.resolve(import.meta.dirname, '..', 'src', 'generated', 'think-memory.generated.js'); + +const source = `/* @generated from contracts/think-memory.graphql by Wesley and Think's v19 renderer. */ + +import { + createObserver, + intent, + reading, +} from '@git-stunts/git-warp/advanced'; + +const MEMORY_DOCUMENT_KEY = 'think.memory-document.v1'; +const MEMORY_INDEX_KEY = 'think.memory-index.v1'; +const MEMORY_INDEX_PAGE_KEY = 'think.memory-index-page.v1'; + +class ThinkMemorySdkValidationError extends TypeError { + constructor(message) { + super(message); + this.name = 'ThinkMemorySdkValidationError'; + } +} + +function requireRequest(request, operation) { + if (typeof request !== 'object' || request === null || Array.isArray(request)) { + throw new ThinkMemorySdkValidationError(\`\${operation} request must be an object\`); + } + return request; +} + +function requireString(value, field) { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new ThinkMemorySdkValidationError(\`\${field} must be a non-empty string\`); + } + return value; +} + +function requireBytes(value, field = 'Think bytes Observer') { + if (value === null || value instanceof Uint8Array) { + return value; + } + if ( + typeof value === 'object' + && value !== null + && typeof value.toUint8Array === 'function' + ) { + return value.toUint8Array(); + } + throw new ThinkMemorySdkValidationError(\`\${field} expected a byte value or null\`); +} + +function requireObservedBoolean(value) { + if (typeof value !== 'boolean') { + throw new ThinkMemorySdkValidationError('Think existence Observer expected a boolean'); + } + return value; +} + +function storeBytes(fields, operation, key) { + const request = requireRequest(fields, operation); + const value = requireBytes(request.value, \`\${operation}.value\`); + if (value === null) { + throw new ThinkMemorySdkValidationError(\`\${operation}.value must be Uint8Array\`); + } + return intent.property.set({ + subject: requireString(request.subject, \`\${operation}.subject\`), + key, + value, + }); +} + +function declareMemoryObject(fields) { + const request = requireRequest(fields, 'thinkMemory.declareMemoryObject'); + return intent.node.add({ + subject: requireString(request.subject, 'thinkMemory.declareMemoryObject.subject'), + }); +} + +function storeMemoryDocument(fields) { + return storeBytes(fields, 'thinkMemory.storeMemoryDocument', MEMORY_DOCUMENT_KEY); +} + +function storeMemoryIndex(fields) { + return storeBytes(fields, 'thinkMemory.storeMemoryIndex', MEMORY_INDEX_KEY); +} + +function storeMemoryIndexPage(fields) { + return storeBytes(fields, 'thinkMemory.storeMemoryIndexPage', MEMORY_INDEX_PAGE_KEY); +} + +function observeBytes(fields, operation, id, key) { + const request = requireRequest(fields, operation); + return createObserver( + id, + reading.property({ + subject: requireString(request.subject, \`\${operation}.subject\`), + key, + }), + requireBytes, + ); +} + +function memoryDocument(fields) { + return observeBytes( + fields, + 'thinkMemory.memoryDocument', + 'think.memory.document', + MEMORY_DOCUMENT_KEY, + ); +} + +function memoryIndex(fields) { + return observeBytes( + fields, + 'thinkMemory.memoryIndex', + 'think.memory.index', + MEMORY_INDEX_KEY, + ); +} + +function memoryIndexPage(fields) { + return observeBytes( + fields, + 'thinkMemory.memoryIndexPage', + 'think.memory.index-page', + MEMORY_INDEX_PAGE_KEY, + ); +} + +function memoryObjectExists(fields) { + const request = requireRequest(fields, 'thinkMemory.memoryObjectExists'); + return createObserver( + 'think.memory.object-exists', + reading.node.exists({ + subject: requireString(request.subject, 'thinkMemory.memoryObjectExists.subject'), + }), + requireObservedBoolean, + ); +} + +export const thinkMemory = Object.freeze({ + intents: Object.freeze({ + declareMemoryObject, + storeMemoryDocument, + storeMemoryIndex, + storeMemoryIndexPage, + }), + observers: Object.freeze({ + memoryDocument, + memoryIndex, + memoryIndexPage, + memoryObjectExists, + }), +}); +`; + +await writeFile(output, source); + +function intentPropertyFields() { + return [ + 'operationType', + 'fieldName', + 'directives.intent.kind', + 'directives.intent.subject', + 'directives.intent.property', + 'directives.intent.value', + ]; +} + +function observerPropertyFields() { + return [ + 'operationType', + 'fieldName', + 'directives.observer.id', + 'directives.observer.reading', + 'directives.observer.subject', + 'directives.observer.property', + 'directives.observer.cardinality', + 'directives.observer.decoder', + ]; +} + +function readPath(value, field) { + return field.split('.').reduce((current, part) => current?.[part], value); +} diff --git a/scripts/repair-v17-mind.mjs b/scripts/repair-v17-mind.mjs deleted file mode 100644 index a1746a7a..00000000 --- a/scripts/repair-v17-mind.mjs +++ /dev/null @@ -1,907 +0,0 @@ -#!/usr/bin/env node - -import { spawnSync } from 'node:child_process'; -import { existsSync, readFileSync } from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; -import { createRequire } from 'node:module'; - -import { createThinkPlumbing } from '../src/git.js'; - -const GRAPH_DEFAULT = 'think'; -const NULL_OID = '0000000000000000000000000000000000000000'; -const CONTENT_ANCHOR_RE = /^(?100644|040000) (?blob|tree) (?[0-9a-f]{40,64})\t_content_(?[0-9a-f]{40,64})$/u; -const requireFromScript = createRequire(import.meta.url); - -export class RepairV17MindError extends Error { - constructor(message, { code = 'repair_v17_mind.error', details = {} } = {}) { - super(message); - this.name = 'RepairV17MindError'; - this.code = code; - this.details = details; - } -} - -export function usage() { - return [ - 'Usage:', - ' node scripts/repair-v17-mind.mjs --mind [--graph think] [--dry-run] [--json]', - ' node scripts/repair-v17-mind.mjs --repo [--graph think] [--dry-run] [--json]', - '', - 'Options:', - ' --mind Resolve a local mind under ~/.think/. The special name "default" maps to ~/.think/repo.', - ' --repo Explicit Think mind Git repository path.', - ' --graph Graph to repair. Defaults to think.', - ' --dry-run Inspect and report without changing refs or checkpoints.', - ' --json Emit machine-readable JSON.', - ' --help, -h Show this help.', - ].join('\n'); -} - -export function parseRepairArgs(argv) { - const args = { - dryRun: false, - graph: GRAPH_DEFAULT, - help: false, - json: false, - mind: null, - repo: null, - }; - - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === '--mind') { - args.mind = requireValue(argv, index, '--mind'); - index += 1; - continue; - } - if (arg === '--repo') { - args.repo = requireValue(argv, index, '--repo'); - index += 1; - continue; - } - if (arg === '--graph') { - args.graph = requireValue(argv, index, '--graph'); - index += 1; - continue; - } - if (arg === '--dry-run') { - args.dryRun = true; - continue; - } - if (arg === '--json') { - args.json = true; - continue; - } - if (arg === '--help' || arg === '-h') { - args.help = true; - continue; - } - throw new RepairV17MindError(`Unknown argument: ${arg ?? ''}`, { - code: 'repair_v17_mind.usage', - }); - } - - return Object.freeze(args); -} - -function requireValue(argv, index, flag) { - const value = argv[index + 1]; - if (value === undefined || value.startsWith('--')) { - throw new RepairV17MindError(`${flag} requires a value`, { - code: 'repair_v17_mind.usage', - }); - } - return value; -} - -export function resolveRepairTarget(args, { - cwd = process.cwd(), - homeDir = process.env.HOME || os.homedir(), -} = {}) { - if (args.mind !== null && args.repo !== null) { - throw new RepairV17MindError('Use either --mind or --repo, not both', { - code: 'repair_v17_mind.usage', - }); - } - - if (args.mind === null && args.repo === null) { - throw new RepairV17MindError('Repair target required: pass --mind or --repo ', { - code: 'repair_v17_mind.usage', - }); - } - - const graph = normalizeGraphName(args.graph); - if (args.repo !== null) { - return Object.freeze({ - graph, - mind: null, - repoDir: path.resolve(cwd, args.repo), - }); - } - - const mind = normalizeMindName(args.mind); - const mindDirName = mind === 'default' ? 'repo' : mind; - return Object.freeze({ - graph, - mind, - repoDir: path.join(homeDir, '.think', mindDirName), - }); -} - -function normalizeGraphName(graph) { - const value = String(graph ?? '').trim(); - if (value.length === 0) { - throw new RepairV17MindError('--graph must not be empty', { - code: 'repair_v17_mind.usage', - }); - } - if (value.includes('/') || value.includes('\\')) { - throw new RepairV17MindError('--graph must be a graph name, not a path', { - code: 'repair_v17_mind.usage', - }); - } - return value; -} - -function normalizeMindName(mind) { - const value = String(mind ?? '').trim(); - if (value.length === 0) { - throw new RepairV17MindError('--mind must not be empty', { - code: 'repair_v17_mind.usage', - }); - } - if (value.includes('/') || value.includes('\\') || value === '.' || value === '..') { - throw new RepairV17MindError('--mind must be one local mind name, not a path', { - code: 'repair_v17_mind.usage', - }); - } - return value; -} - -export async function repairV17Mind(options, deps = {}) { - const target = resolveRepairTarget(options, deps); - const runner = deps.runner ?? runCommand; - const now = deps.now ?? new Date(); - const runUpgrade = deps.runUpgrade ?? runGitWarpUpgrade; - const materialize = deps.materialize ?? materializeGraph; - const runCheck = deps.runCheck ?? checkGraph; - const runDoctor = deps.runDoctor ?? doctorGraph; - - assertGitRepo(target.repoDir, runner); - - const checkpointRef = buildCheckpointRef(target.graph); - const beforeCheckpoint = readCheckpointSha(target.repoDir, checkpointRef, runner); - const upgradeBefore = await runUpgrade({ - graph: target.graph, - repoDir: target.repoDir, - dryRun: true, - runner, - }); - - if (options.dryRun) { - return Object.freeze({ - ok: true, - dryRun: true, - repo: target.repoDir, - graph: target.graph, - checkpointRef, - beforeCheckpoint, - backupRef: null, - changed: false, - upgrade: { - before: normalizeUpgradeStatus(upgradeBefore), - after: null, - rawBefore: upgradeBefore, - rawAfter: null, - }, - wouldRepair: upgradeBefore.status === 'would-upgrade', - }); - } - - if (upgradeBefore.status !== 'would-upgrade') { - return Object.freeze({ - ok: true, - dryRun: false, - repo: target.repoDir, - graph: target.graph, - checkpointRef, - beforeCheckpoint, - backupRef: null, - changed: false, - upgrade: { - before: normalizeUpgradeStatus(upgradeBefore), - after: normalizeUpgradeStatus(upgradeBefore), - rawBefore: upgradeBefore, - rawAfter: upgradeBefore, - }, - materialize: null, - check: null, - doctor: null, - }); - } - - if (beforeCheckpoint === null) { - throw new RepairV17MindError('Upgrade dry-run reported work, but no checkpoint ref exists to back up', { - code: 'repair_v17_mind.inconsistent_checkpoint', - details: { checkpointRef, repo: target.repoDir }, - }); - } - - const backupRef = createBackupRef({ - checkpointSha: beforeCheckpoint, - graph: target.graph, - now, - repoDir: target.repoDir, - runner, - }); - - const upgradeActual = await runUpgrade({ - graph: target.graph, - repoDir: target.repoDir, - dryRun: false, - runner, - }); - const materializeResult = await materialize({ - graph: target.graph, - repoDir: target.repoDir, - runner, - }); - const upgradeAfter = await runUpgrade({ - graph: target.graph, - repoDir: target.repoDir, - dryRun: true, - runner, - }); - const checkResult = await runCheck({ - graph: target.graph, - repoDir: target.repoDir, - runner, - }); - const doctorResult = await runDoctor({ - graph: target.graph, - repoDir: target.repoDir, - runner, - }); - - validatePostRepair({ - checkResult, - doctorResult, - upgradeAfter, - }); - - return Object.freeze({ - ok: true, - dryRun: false, - repo: target.repoDir, - graph: target.graph, - checkpointRef, - backupRef, - beforeCheckpoint, - afterCheckpoint: resolveAfterCheckpoint({ - checkResult, - materializeResult, - upgradeActual, - }), - changed: true, - upgrade: { - before: normalizeUpgradeStatus(upgradeBefore), - after: normalizeUpgradeStatus(upgradeAfter), - rawBefore: upgradeBefore, - rawActual: upgradeActual, - rawAfter: upgradeAfter, - }, - materialize: materializeResult, - check: { - patchesSinceCheckpoint: checkResult.status?.patchesSinceCheckpoint ?? null, - raw: checkResult, - }, - doctor: { - failures: doctorResult.summary?.fail ?? null, - warnings: collectDoctorWarnings(doctorResult), - raw: doctorResult, - }, - }); -} - -function assertGitRepo(repoDir, runner) { - const result = runGit(repoDir, ['rev-parse', '--git-dir'], runner, { allowExitCodes: [0, 128] }); - if (result.status === 0) { - return; - } - - throw new RepairV17MindError(`Repair target is not a Git repository: ${repoDir}`, { - code: 'repair_v17_mind.repo_not_found', - details: { repo: repoDir }, - }); -} - -export function buildCheckpointRef(graph) { - return `refs/warp/${graph}/checkpoints/head`; -} - -function readCheckpointSha(repoDir, checkpointRef, runner) { - const result = runGit(repoDir, ['rev-parse', '--verify', '--quiet', checkpointRef], runner, { - allowExitCodes: [0, 1], - }); - if (result.status === 0) { - return result.stdout.trim(); - } - return null; -} - -export function createBackupRef({ checkpointSha, graph, now, repoDir, runner = runCommand }) { - for (let index = 0; index < 100; index += 1) { - const backupRef = buildBackupRef(graph, now, index); - if (refExists(repoDir, backupRef, runner)) { - continue; - } - - runGit(repoDir, ['update-ref', '--create-reflog', backupRef, checkpointSha, NULL_OID], runner); - return backupRef; - } - - throw new RepairV17MindError('Could not allocate a unique pre-v17 checkpoint backup ref', { - code: 'repair_v17_mind.backup_ref_exhausted', - details: { graph, repo: repoDir }, - }); -} - -function buildBackupRef(graph, now, index) { - const suffix = index === 0 ? '' : `-${String(index + 1).padStart(2, '0')}`; - return `refs/warp/${graph}/checkpoints/pre-v17-upgrade-${formatBackupTimestamp(now)}${suffix}`; -} - -function refExists(repoDir, ref, runner) { - const result = runGit(repoDir, ['show-ref', '--verify', '--quiet', ref], runner, { - allowExitCodes: [0, 1], - }); - return result.status === 0; -} - -export function formatBackupTimestamp(date) { - const parts = [ - date.getUTCFullYear(), - date.getUTCMonth() + 1, - date.getUTCDate(), - date.getUTCHours(), - date.getUTCMinutes(), - date.getUTCSeconds(), - ]; - const [year, month, day, hour, minute, second] = parts.map(part => String(part).padStart(2, '0')); - return `${year}${month}${day}-${hour}${minute}${second}`; -} - -export async function runGitWarpUpgrade({ repoDir, graph, dryRun, runner = runCommand }) { - const packageRoot = resolveGitWarpPackageRoot(); - const upgradeModule = await importGitWarpUpgradeModule(packageRoot); - const persistence = await createV17Persistence({ repoDir, runner }); - const cryptoAdapter = await createV17Crypto(packageRoot); - - return await upgradeModule.upgradeCheckpointSchema({ - persistence, - graphName: graph, - dryRun, - crypto: cryptoAdapter, - }); -} - -async function importGitWarpUpgradeModule(packageRoot) { - return await importGitWarpDistModule(packageRoot, [ - 'scripts', - 'migrations', - 'v17.0.0', - 'checkpoint-schema-upgrade.js', - ], { - code: 'repair_v17_mind.v17_upgrade_unavailable', - message: 'Installed @git-stunts/git-warp does not expose the v17 checkpoint migration script', - }); -} - -async function importGitWarpDistModule(packageRoot, relativeParts, { code, message }) { - const modulePath = path.join(packageRoot, 'dist', ...relativeParts); - if (!existsSync(modulePath)) { - throw new RepairV17MindError(message, { - code, - details: { modulePath }, - }); - } - - return await import(pathToFileURL(modulePath).href); -} - -async function createV17Crypto(packageRoot) { - const cryptoPath = path.join( - packageRoot, - 'dist', - 'src', - 'infrastructure', - 'adapters', - 'NodeCryptoAdapter.js' - ); - if (!existsSync(cryptoPath)) { - return undefined; - } - - const cryptoModule = await import(pathToFileURL(cryptoPath).href); - const NodeCryptoAdapter = cryptoModule.default ?? cryptoModule.NodeCryptoAdapter; - if (typeof NodeCryptoAdapter !== 'function') { - return undefined; - } - return new NodeCryptoAdapter(); -} - -async function createV17Persistence({ repoDir, runner }) { - const gitWarp = await import('@git-stunts/git-warp'); - const { GitGraphAdapter } = gitWarp; - const persistence = new GitGraphAdapter({ - plumbing: createThinkPlumbing(repoDir), - }); - - return createContentAnchorAwarePersistence( - persistence, - createGitObjectTypeReader({ repoDir, runner }) - ); -} - -export async function materializeGraph({ repoDir, graph, runner = runCommand }) { - const gitWarp = await import('@git-stunts/git-warp'); - const { default: DefaultWarpApp, GitGraphAdapter, WarpApp: NamedWarpApp } = gitWarp; - const packageRoot = resolveGitWarpPackageRoot(); - const WarpApp = DefaultWarpApp ?? NamedWarpApp; - const persistence = createContentAnchorAwarePersistence( - new GitGraphAdapter({ - plumbing: createThinkPlumbing(repoDir), - }), - createGitObjectTypeReader({ repoDir, runner }) - ); - const patchJournal = await createV17PatchJournal({ packageRoot, persistence }); - const app = await WarpApp.open({ - persistence, - graphName: graph, - writerId: 'think-repair-v17', - checkpointPolicy: { every: 100 }, - patchJournal, - }); - const state = await app.core().materialize(); - const checkpoint = await app.core().createCheckpoint(); - - return Object.freeze({ - graph, - checkpoint, - edges: sizeOf(state?.edgeAlive), - nodes: sizeOf(state?.nodeAlive), - properties: sizeOf(state?.prop), - }); -} - -async function createV17PatchJournal({ packageRoot, persistence }) { - const modules = await importV17PatchJournalModules(packageRoot); - const journalOptions = await createV17PatchJournalOptions({ modules, persistence }); - return new modules.CborPatchJournalAdapter(journalOptions); -} - -async function importV17PatchJournalModules(packageRoot) { - const [ - codecModule, - journalModule, - policyModule, - ] = await Promise.all([ - importV17CborCodecModule(packageRoot), - importV17PatchJournalModule(packageRoot), - importV17CompatibilityPolicyModule(packageRoot), - ]); - - return Object.freeze({ - CborPatchJournalAdapter: journalModule.CborPatchJournalAdapter, - codec: codecModule.default, - compatibilityPolicy: policyModule.V17_SUBSTRATE_MIGRATION_COMPATIBILITY_POLICY, - }); -} - -function importV17CborCodecModule(packageRoot) { - return importGitWarpDistModule(packageRoot, [ - 'src', - 'infrastructure', - 'codecs', - 'CborCodec.js', - ], { - code: 'repair_v17_mind.codec_unavailable', - message: 'Installed @git-stunts/git-warp does not expose the CBOR codec', - }); -} - -function importV17PatchJournalModule(packageRoot) { - return importGitWarpDistModule(packageRoot, [ - 'src', - 'infrastructure', - 'adapters', - 'CborPatchJournalAdapter.js', - ], { - code: 'repair_v17_mind.patch_journal_unavailable', - message: 'Installed @git-stunts/git-warp does not expose the CBOR patch journal adapter', - }); -} - -function importV17CompatibilityPolicyModule(packageRoot) { - return importGitWarpDistModule(packageRoot, [ - 'scripts', - 'migrations', - 'v17.0.0', - 'SubstrateMigrationCompatibilityPolicy.js', - ], { - code: 'repair_v17_mind.compatibility_policy_unavailable', - message: 'Installed @git-stunts/git-warp does not expose the v17 substrate compatibility policy', - }); -} - -async function createV17PatchJournalOptions({ modules, persistence }) { - const writeStorage = typeof persistence.defaultPatchWriteStorage === 'function' - ? persistence.defaultPatchWriteStorage() - : undefined; - const blobStorage = await createV17PatchBlobStorage(persistence, writeStorage); - const journalOptions = { - codec: modules.codec, - blobPort: persistence, - commitPort: persistence, - compatibilityPolicy: modules.compatibilityPolicy, - }; - if (writeStorage !== undefined) { - journalOptions.writeStorage = writeStorage; - } - if (blobStorage !== null) { - journalOptions.blobStorage = blobStorage; - } - - return journalOptions; -} - -async function createV17PatchBlobStorage(persistence, writeStorage) { - if (writeStorage?.strategy === 'git-cas' && typeof persistence.createRuntimeBlobStorage === 'function') { - return await persistence.createRuntimeBlobStorage(); - } - return null; -} - -function sizeOf(value) { - if (typeof value?.size === 'number') { - return value.size; - } - if (typeof value?.count === 'number') { - return value.count; - } - return null; -} - -export function checkGraph({ repoDir, graph, runner = runCommand }) { - return runGitWarpJsonCommand({ - command: 'check', - graph, - repoDir, - runner, - }); -} - -export function doctorGraph({ repoDir, graph, runner = runCommand }) { - return runGitWarpJsonCommand({ - command: 'doctor', - graph, - repoDir, - runner, - allowExitCodes: [0, 3], - }); -} - -function runGitWarpJsonCommand({ - allowExitCodes = [0], - command, - graph, - repoDir, - runner, -}) { - const gitWarp = resolveGitWarpBinCommand(); - const result = runner(gitWarp.command, [ - ...gitWarp.args, - '--repo', - repoDir, - '--graph', - graph, - '--json', - command, - ], { - allowExitCodes, - }); - return parseJsonCommandOutput(result.stdout, command); -} - -export function createContentAnchorAwarePersistence(persistence, readObjectType) { - return new Proxy(persistence, { - get(target, property, receiver) { - if (property === 'writeTree') { - return async (entries) => { - const normalized = await normalizeContentAnchorTreeEntries(entries, readObjectType); - return await target.writeTree(normalized); - }; - } - - const value = Reflect.get(target, property, receiver); - if (typeof value === 'function') { - return value.bind(target); - } - return value; - }, - }); -} - -export async function normalizeContentAnchorTreeEntries(entries, readObjectType) { - return await Promise.all(entries.map(async (entry) => { - const parsed = parseContentAnchorEntry(entry); - if (parsed === null) { - return entry; - } - - const objectType = await readObjectType(parsed.oid); - if (objectType === 'blob') { - return `100644 blob ${parsed.oid}\t_content_${parsed.oid}`; - } - if (objectType === 'tree') { - return `040000 tree ${parsed.oid}\t_content_${parsed.oid}`; - } - - throw new RepairV17MindError(`Unsupported content anchor object type: ${objectType}`, { - code: 'repair_v17_mind.unsupported_content_anchor_type', - details: { objectType, oid: parsed.oid }, - }); - })); -} - -function parseContentAnchorEntry(entry) { - const match = CONTENT_ANCHOR_RE.exec(entry); - if (match === null) { - return null; - } - - const groups = match.groups ?? {}; - if (groups.oid !== groups.anchorOid) { - return null; - } - - return Object.freeze({ - kind: groups.kind, - mode: groups.mode, - oid: groups.oid, - }); -} - -function createGitObjectTypeReader({ repoDir, runner }) { - return (oid) => { - const result = runGit(repoDir, ['cat-file', '-t', oid], runner); - return result.stdout.trim(); - }; -} - -export function resolveGitWarpPackageRoot(packageName = '@git-stunts/git-warp') { - const packageEntry = requireFromScript.resolve(packageName); - let dir = path.dirname(packageEntry); - while (dir !== path.dirname(dir)) { - const packageJsonPath = path.join(dir, 'package.json'); - if (existsSync(packageJsonPath)) { - const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8')); - if (packageJson.name === packageName) { - return dir; - } - } - dir = path.dirname(dir); - } - - throw new RepairV17MindError(`Could not resolve ${packageName} package root`, { - code: 'repair_v17_mind.package_root_unavailable', - details: { packageName }, - }); -} - -function resolveGitWarpBinCommand() { - const packageRoot = resolveGitWarpPackageRoot(); - const packageJson = JSON.parse(readFileSync(path.join(packageRoot, 'package.json'), 'utf8')); - const bin = typeof packageJson.bin === 'string' - ? packageJson.bin - : packageJson.bin?.['git-warp']; - const relativeBin = bin ?? 'bin/git-warp'; - const binPath = path.join(packageRoot, relativeBin); - if (!existsSync(binPath)) { - throw new RepairV17MindError('Installed @git-stunts/git-warp does not expose a git-warp binary', { - code: 'repair_v17_mind.git_warp_bin_unavailable', - details: { binPath }, - }); - } - - return Object.freeze({ - command: process.execPath, - args: [binPath], - }); -} - -function runGit(repoDir, args, runner, { allowExitCodes = [0] } = {}) { - return runner('git', ['-C', repoDir, ...args], { - allowExitCodes, - }); -} - -export function runCommand(command, args, { - allowExitCodes = [0], - cwd = process.cwd(), - env = {}, -} = {}) { - const result = spawnSync(command, args, { - cwd, - encoding: 'utf8', - env: { - ...process.env, - NO_COLOR: '1', - ...env, - }, - }); - if (result.error) { - throw new RepairV17MindError(result.error.message, { - code: 'repair_v17_mind.command_error', - details: { args, command }, - }); - } - - const status = result.status ?? 1; - if (!allowExitCodes.includes(status)) { - throw new RepairV17MindError(`Command failed: ${command} ${args.join(' ')}`, { - code: 'repair_v17_mind.command_failed', - details: { - args, - command, - status, - stderr: result.stderr ?? '', - stdout: result.stdout ?? '', - }, - }); - } - - return Object.freeze({ - args, - command, - status, - stderr: result.stderr ?? '', - stdout: result.stdout ?? '', - }); -} - -function parseJsonCommandOutput(stdout, label) { - try { - return JSON.parse(stdout); - } catch (error) { - throw new RepairV17MindError(`Could not parse JSON output from ${label}`, { - code: 'repair_v17_mind.invalid_json', - details: { - cause: error instanceof Error ? error.message : String(error), - stdout, - }, - }); - } -} - -function normalizeUpgradeStatus(result) { - if (result.status === 'would-upgrade') { - return 'needed'; - } - return result.status; -} - -function validatePostRepair({ checkResult, doctorResult, upgradeAfter }) { - if (upgradeAfter.status !== 'already-current') { - throw new RepairV17MindError('Post-repair upgrade dry-run did not report already-current', { - code: 'repair_v17_mind.upgrade_not_current', - details: { upgradeAfter }, - }); - } - - const patchesSinceCheckpoint = checkResult.status?.patchesSinceCheckpoint; - if (patchesSinceCheckpoint !== 0) { - throw new RepairV17MindError('Post-repair git-warp check reported patches since checkpoint', { - code: 'repair_v17_mind.check_not_fresh', - details: { patchesSinceCheckpoint }, - }); - } - - const failures = doctorResult.summary?.fail; - if (failures !== 0) { - throw new RepairV17MindError('Post-repair git-warp doctor reported failures', { - code: 'repair_v17_mind.doctor_failures', - details: { failures, doctorResult }, - }); - } -} - -function resolveAfterCheckpoint({ checkResult, materializeResult, upgradeActual }) { - return checkResult.checkpoint?.sha - ?? materializeResult.checkpoint - ?? upgradeActual.upgradedCheckpointSha - ?? null; -} - -function collectDoctorWarnings(doctorResult) { - const findings = Array.isArray(doctorResult.findings) ? doctorResult.findings : []; - return findings.filter(finding => finding?.status === 'warn'); -} - -function formatHumanResult(result) { - if (result.dryRun) { - return [ - `Repo: ${result.repo}`, - `Graph: ${result.graph}`, - `Checkpoint: ${result.beforeCheckpoint ?? '(missing)'}`, - `Upgrade: ${result.upgrade.before}`, - `Would repair: ${String(result.wouldRepair)}`, - ].join('\n'); - } - - if (!result.changed) { - return [ - `Repo: ${result.repo}`, - `Graph: ${result.graph}`, - `No repair needed: ${result.upgrade.before}`, - ].join('\n'); - } - - return [ - `Repaired ${result.repo} graph ${result.graph}.`, - `Backup: ${result.backupRef}`, - `Before: ${result.beforeCheckpoint}`, - `After: ${result.afterCheckpoint ?? '(unknown)'}`, - `Patches since checkpoint: ${String(result.check.patchesSinceCheckpoint)}`, - `Doctor failures: ${String(result.doctor.failures)}`, - `Doctor warnings: ${String(result.doctor.warnings.length)}`, - ].join('\n'); -} - -async function main() { - const args = parseRepairArgs(process.argv.slice(2)); - if (args.help) { - process.stdout.write(`${usage()}\n`); - return; - } - - const result = await repairV17Mind(args); - if (args.json) { - process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); - return; - } - - process.stdout.write(`${formatHumanResult(result)}\n`); -} - -function isMainModule() { - const invokedPath = process.argv[1]; - if (invokedPath === undefined) { - return false; - } - - return path.resolve(invokedPath) === fileURLToPath(import.meta.url); -} - -if (isMainModule()) { - main().catch((error) => { - const message = error instanceof Error ? error.message : String(error); - const code = error instanceof RepairV17MindError ? error.code : 'repair_v17_mind.internal'; - const details = error instanceof RepairV17MindError ? error.details : {}; - if (process.argv.includes('--json')) { - process.stdout.write(`${JSON.stringify({ - ok: false, - error: { - code, - message, - details, - }, - }, null, 2)}\n`); - } else { - process.stderr.write(`${message}\n\n${usage()}\n`); - } - process.exitCode = 1; - }); -} diff --git a/src/browse-benchmark.js b/src/browse-benchmark.js index ec919817..386d70cb 100644 --- a/src/browse-benchmark.js +++ b/src/browse-benchmark.js @@ -1,9 +1,6 @@ -import WarpApp, { GitGraphAdapter } from '@git-stunts/git-warp'; - import { ValidationError } from './errors.js'; -import { createThinkPlumbing, ensureGitRepo, hasGitRepo } from './git.js'; +import { ensureGitRepo, hasGitRepo } from './git.js'; import { - GRAPH_NAME, loadBrowseChronologyEntries, prepareBrowseBootstrap as loadBrowseBootstrap, } from './store.js'; @@ -13,9 +10,10 @@ import { GRAPH_MODEL_VERSION, SCHEMA_VERSION, SESSION_PREFIX, - TEXT_MIME, } from './store/constants.js'; -import { encodeTextContent } from './store/content.js'; +import { appendIndexedMemoryObject } from './store/native-index.js'; +import { encodeNativeDocument } from './store/native-document.js'; +import { openNativeMemory } from './store/native-runtime.js'; const DEFAULT_START_TIME_MS = Date.parse('2026-03-20T16:00:00.000Z'); const WITHIN_SESSION_GAP_MS = 30 * 1000; @@ -71,7 +69,9 @@ export async function createSyntheticBrowseFixture({ const capturesPerSession = distributeCaptures(captureCount, sessionCount); let currentMs = DEFAULT_START_TIME_MS; let createdCaptures = 0; - const graph = await openGraph(repoDir); + const memory = await openNativeMemory(repoDir, { + writerId: 'benchmark-fixture', + }); const entries = []; for (let sessionIndex = 0; sessionIndex < capturesPerSession.length; sessionIndex += 1) { @@ -109,49 +109,48 @@ export async function createSyntheticBrowseFixture({ } } - await graph.patch(async (patch) => { - patch - .addNode(GRAPH_META_ID) - .setProperty(GRAPH_META_ID, 'kind', 'graph_meta') - .setProperty(GRAPH_META_ID, 'createdAt', new Date(DEFAULT_START_TIME_MS).toISOString()) - .setProperty(GRAPH_META_ID, 'updatedAt', new Date(currentMs).toISOString()) - .setProperty(GRAPH_META_ID, 'graphModelVersion', GRAPH_MODEL_VERSION); - - for (const item of entries) { - if (item.type === 'session') { - patch - .addNode(item.id) - .setProperty(item.id, 'kind', 'session') - .setProperty(item.id, 'createdAt', item.createdAt) - .setProperty(item.id, 'sortKey', item.sortKey) - .setProperty(item.id, 'schemaVersion', SCHEMA_VERSION); - continue; - } - - patch - .addNode(item.id) - .setProperty(item.id, 'kind', 'capture') - .setProperty(item.id, 'source', 'benchmark') - .setProperty(item.id, 'channel', 'benchmark') - .setProperty(item.id, 'writerId', graph.writerId) - .setProperty(item.id, 'createdAt', item.createdAt) - .setProperty(item.id, 'sortKey', item.sortKey) - .setProperty(item.id, 'sessionId', item.sessionId); - - patch.addEdge(item.id, item.sessionId, 'captured_in'); - - // eslint-disable-next-line no-await-in-loop -- sequential graph writes within a patch transaction - await patch.attachContent(item.id, encodeTextContent(item.text), { mime: TEXT_MIME }); - } - - const captures = entries - .filter((item) => item.type === 'capture') - .sort((left, right) => right.sortKey.localeCompare(left.sortKey)); + await memory.writeMemoryDocument({ + id: GRAPH_META_ID, + bytes: encodeNativeDocument('memory-object', { + id: GRAPH_META_ID, + kind: 'graph_meta', + createdAt: new Date(DEFAULT_START_TIME_MS).toISOString(), + updatedAt: new Date(currentMs).toISOString(), + graphModelVersion: GRAPH_MODEL_VERSION, + }), + }); - for (let index = 0; index + 1 < captures.length; index += 1) { - patch.addEdge(captures[index].id, captures[index + 1].id, 'older'); + for (const item of entries) { + if (item.type === 'session') { + // eslint-disable-next-line no-await-in-loop -- benchmark fixture construction is intentionally sequential + await appendIndexedMemoryObject(repoDir, { + id: item.id, + kind: 'session', + facts: { + kind: 'session', + createdAt: item.createdAt, + sortKey: item.sortKey, + schemaVersion: SCHEMA_VERSION, + }, + }); + continue; } - }); + // eslint-disable-next-line no-await-in-loop -- benchmark fixture construction is intentionally sequential + await appendIndexedMemoryObject(repoDir, { + id: item.id, + kind: 'capture', + facts: { + kind: 'capture', + text: item.text, + source: 'benchmark', + channel: 'benchmark', + writerId: memory.writerId, + createdAt: item.createdAt, + sortKey: item.sortKey, + sessionId: item.sessionId, + }, + }); + } return Object.freeze({ captureCount, @@ -196,15 +195,3 @@ function createSyntheticEntry({ thoughtNumber, sessionNumber, captureNumberInSes sessionId: null, }; } - -// eslint-disable-next-line require-await -- wraps git-warp WarpApp.open which returns a promise -async function openGraph(repoDir) { - const plumbing = createThinkPlumbing(repoDir); - const persistence = new GitGraphAdapter({ plumbing }); - - return WarpApp.open({ - persistence, - graphName: GRAPH_NAME, - writerId: 'benchmark-fixture', - }); -} diff --git a/src/cli/commands/capture.js b/src/cli/commands/capture.js index 508cd6c6..0418e520 100644 --- a/src/cli/commands/capture.js +++ b/src/cli/commands/capture.js @@ -1,6 +1,6 @@ import { ensureGitRepo, hasGitRepo, pushWarpRefs } from '../../git.js'; import { captureProvenanceFromEnvironment } from '../../capture-provenance.js'; -import { getCaptureAmbientContext, getAmbientProjectContext } from '../../project-context.js'; +import { getAmbientProjectContext } from '../../project-context.js'; import { getLocalRepoDir, getUpstreamUrl } from '../../paths.js'; import { finalizeCapturedThought, @@ -36,10 +36,14 @@ export async function runCapture(thought, output, reporter) { reporter.event(repoAlreadyExists ? 'repo.ensure.done' : 'repo.bootstrap.done', { repoDir }); const provenance = captureProvenanceFromEnvironment(process.env); - const ambientContext = getCaptureAmbientContext(process.cwd()); + const ambientContext = getAmbientProjectContext(process.cwd()); reporter.event('capture.local_save.start'); - const entry = await saveRawCapture(repoDir, thought, { provenance, ambientContext }); + const entry = await saveRawCapture(repoDir, thought, { + provenance, + ambientContext, + initializeGraphModel: !repoAlreadyExists, + }); reporter.event('capture.local_save.done', { entryId: entry.id }); output.out('Saved locally', 'capture.status', { diff --git a/src/generated/think-memory.generated.js b/src/generated/think-memory.generated.js new file mode 100644 index 00000000..b35db3ad --- /dev/null +++ b/src/generated/think-memory.generated.js @@ -0,0 +1,150 @@ +/* @generated from contracts/think-memory.graphql by Wesley and Think's v19 renderer. */ + +import { + createObserver, + intent, + reading, +} from '@git-stunts/git-warp/advanced'; + +const MEMORY_DOCUMENT_KEY = 'think.memory-document.v1'; +const MEMORY_INDEX_KEY = 'think.memory-index.v1'; +const MEMORY_INDEX_PAGE_KEY = 'think.memory-index-page.v1'; + +class ThinkMemorySdkValidationError extends TypeError { + constructor(message) { + super(message); + this.name = 'ThinkMemorySdkValidationError'; + } +} + +function requireRequest(request, operation) { + if (typeof request !== 'object' || request === null || Array.isArray(request)) { + throw new ThinkMemorySdkValidationError(`${operation} request must be an object`); + } + return request; +} + +function requireString(value, field) { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new ThinkMemorySdkValidationError(`${field} must be a non-empty string`); + } + return value; +} + +function requireBytes(value, field = 'Think bytes Observer') { + if (value === null || value instanceof Uint8Array) { + return value; + } + if ( + typeof value === 'object' + && value !== null + && typeof value.toUint8Array === 'function' + ) { + return value.toUint8Array(); + } + throw new ThinkMemorySdkValidationError(`${field} expected a byte value or null`); +} + +function requireObservedBoolean(value) { + if (typeof value !== 'boolean') { + throw new ThinkMemorySdkValidationError('Think existence Observer expected a boolean'); + } + return value; +} + +function storeBytes(fields, operation, key) { + const request = requireRequest(fields, operation); + const value = requireBytes(request.value, `${operation}.value`); + if (value === null) { + throw new ThinkMemorySdkValidationError(`${operation}.value must be Uint8Array`); + } + return intent.property.set({ + subject: requireString(request.subject, `${operation}.subject`), + key, + value, + }); +} + +function declareMemoryObject(fields) { + const request = requireRequest(fields, 'thinkMemory.declareMemoryObject'); + return intent.node.add({ + subject: requireString(request.subject, 'thinkMemory.declareMemoryObject.subject'), + }); +} + +function storeMemoryDocument(fields) { + return storeBytes(fields, 'thinkMemory.storeMemoryDocument', MEMORY_DOCUMENT_KEY); +} + +function storeMemoryIndex(fields) { + return storeBytes(fields, 'thinkMemory.storeMemoryIndex', MEMORY_INDEX_KEY); +} + +function storeMemoryIndexPage(fields) { + return storeBytes(fields, 'thinkMemory.storeMemoryIndexPage', MEMORY_INDEX_PAGE_KEY); +} + +function observeBytes(fields, operation, id, key) { + const request = requireRequest(fields, operation); + return createObserver( + id, + reading.property({ + subject: requireString(request.subject, `${operation}.subject`), + key, + }), + requireBytes, + ); +} + +function memoryDocument(fields) { + return observeBytes( + fields, + 'thinkMemory.memoryDocument', + 'think.memory.document', + MEMORY_DOCUMENT_KEY, + ); +} + +function memoryIndex(fields) { + return observeBytes( + fields, + 'thinkMemory.memoryIndex', + 'think.memory.index', + MEMORY_INDEX_KEY, + ); +} + +function memoryIndexPage(fields) { + return observeBytes( + fields, + 'thinkMemory.memoryIndexPage', + 'think.memory.index-page', + MEMORY_INDEX_PAGE_KEY, + ); +} + +function memoryObjectExists(fields) { + const request = requireRequest(fields, 'thinkMemory.memoryObjectExists'); + return createObserver( + 'think.memory.object-exists', + reading.node.exists({ + subject: requireString(request.subject, 'thinkMemory.memoryObjectExists.subject'), + }), + requireObservedBoolean, + ); +} + +export const thinkMemory = Object.freeze({ + intents: Object.freeze({ + declareMemoryObject, + storeMemoryDocument, + storeMemoryIndex, + storeMemoryIndexPage, + }), + observers: Object.freeze({ + memoryDocument, + memoryIndex, + memoryIndexPage, + memoryObjectExists, + }), +}); diff --git a/src/generated/think-memory.wesley.generated.ts b/src/generated/think-memory.wesley.generated.ts new file mode 100644 index 00000000..0f53face --- /dev/null +++ b/src/generated/think-memory.wesley.generated.ts @@ -0,0 +1,162 @@ +/* @generated by Wesley. Do not edit. */ + +export type Bytes = unknown; + +export interface IntentBinding { + operation: string; +} + +export type IntentKind = "NODE_ADD" | "PROPERTY_SET_BYTES"; + +export type ObserverCardinality = "EXACTLY_ONE"; + +export type ReadingKind = "PROPERTY" | "NODE_EXISTS"; + +export type ValueDecoder = "BOOLEAN" | "BYTES"; + +export interface MutationDeclareMemoryObjectRequest { + subject: string; +} + +export type MutationDeclareMemoryObjectResponse = IntentBinding; + +export const mutationDeclareMemoryObjectOperation = { + operationType: "MUTATION", + fieldName: "declareMemoryObject", + directives: {"intent":{"kind":"NODE_ADD","subject":"subject"}}, +} as const; + +export type MutationDeclareMemoryObjectOperation = { + request: MutationDeclareMemoryObjectRequest; + response: MutationDeclareMemoryObjectResponse; + metadata: typeof mutationDeclareMemoryObjectOperation; +}; + +export interface MutationStoreMemoryDocumentRequest { + subject: string; + value: Bytes; +} + +export type MutationStoreMemoryDocumentResponse = IntentBinding; + +export const mutationStoreMemoryDocumentOperation = { + operationType: "MUTATION", + fieldName: "storeMemoryDocument", + directives: {"intent":{"kind":"PROPERTY_SET_BYTES","subject":"subject","property":"think.memory-document.v1","value":"value"}}, +} as const; + +export type MutationStoreMemoryDocumentOperation = { + request: MutationStoreMemoryDocumentRequest; + response: MutationStoreMemoryDocumentResponse; + metadata: typeof mutationStoreMemoryDocumentOperation; +}; + +export interface MutationStoreMemoryIndexRequest { + subject: string; + value: Bytes; +} + +export type MutationStoreMemoryIndexResponse = IntentBinding; + +export const mutationStoreMemoryIndexOperation = { + operationType: "MUTATION", + fieldName: "storeMemoryIndex", + directives: {"intent":{"kind":"PROPERTY_SET_BYTES","subject":"subject","property":"think.memory-index.v1","value":"value"}}, +} as const; + +export type MutationStoreMemoryIndexOperation = { + request: MutationStoreMemoryIndexRequest; + response: MutationStoreMemoryIndexResponse; + metadata: typeof mutationStoreMemoryIndexOperation; +}; + +export interface MutationStoreMemoryIndexPageRequest { + subject: string; + value: Bytes; +} + +export type MutationStoreMemoryIndexPageResponse = IntentBinding; + +export const mutationStoreMemoryIndexPageOperation = { + operationType: "MUTATION", + fieldName: "storeMemoryIndexPage", + directives: {"intent":{"kind":"PROPERTY_SET_BYTES","subject":"subject","property":"think.memory-index-page.v1","value":"value"}}, +} as const; + +export type MutationStoreMemoryIndexPageOperation = { + request: MutationStoreMemoryIndexPageRequest; + response: MutationStoreMemoryIndexPageResponse; + metadata: typeof mutationStoreMemoryIndexPageOperation; +}; + +export interface QueryMemoryDocumentRequest { + subject: string; +} + +export type QueryMemoryDocumentResponse = Bytes | null; + +export const queryMemoryDocumentOperation = { + operationType: "QUERY", + fieldName: "memoryDocument", + directives: {"observer":{"id":"think.memory.document","reading":"PROPERTY","subject":"subject","property":"think.memory-document.v1","cardinality":"EXACTLY_ONE","decoder":"BYTES"}}, +} as const; + +export type QueryMemoryDocumentOperation = { + request: QueryMemoryDocumentRequest; + response: QueryMemoryDocumentResponse; + metadata: typeof queryMemoryDocumentOperation; +}; + +export interface QueryMemoryIndexRequest { + subject: string; +} + +export type QueryMemoryIndexResponse = Bytes | null; + +export const queryMemoryIndexOperation = { + operationType: "QUERY", + fieldName: "memoryIndex", + directives: {"observer":{"id":"think.memory.index","reading":"PROPERTY","subject":"subject","property":"think.memory-index.v1","cardinality":"EXACTLY_ONE","decoder":"BYTES"}}, +} as const; + +export type QueryMemoryIndexOperation = { + request: QueryMemoryIndexRequest; + response: QueryMemoryIndexResponse; + metadata: typeof queryMemoryIndexOperation; +}; + +export interface QueryMemoryIndexPageRequest { + subject: string; +} + +export type QueryMemoryIndexPageResponse = Bytes | null; + +export const queryMemoryIndexPageOperation = { + operationType: "QUERY", + fieldName: "memoryIndexPage", + directives: {"observer":{"id":"think.memory.index-page","reading":"PROPERTY","subject":"subject","property":"think.memory-index-page.v1","cardinality":"EXACTLY_ONE","decoder":"BYTES"}}, +} as const; + +export type QueryMemoryIndexPageOperation = { + request: QueryMemoryIndexPageRequest; + response: QueryMemoryIndexPageResponse; + metadata: typeof queryMemoryIndexPageOperation; +}; + +export interface QueryMemoryObjectExistsRequest { + subject: string; +} + +export type QueryMemoryObjectExistsResponse = boolean; + +export const queryMemoryObjectExistsOperation = { + operationType: "QUERY", + fieldName: "memoryObjectExists", + directives: {"observer":{"id":"think.memory.object-exists","reading":"NODE_EXISTS","subject":"subject","cardinality":"EXACTLY_ONE","decoder":"BOOLEAN"}}, +} as const; + +export type QueryMemoryObjectExistsOperation = { + request: QueryMemoryObjectExistsRequest; + response: QueryMemoryObjectExistsResponse; + metadata: typeof queryMemoryObjectExistsOperation; +}; diff --git a/src/mcp/service.js b/src/mcp/service.js index cfa4b6db..b108037a 100644 --- a/src/mcp/service.js +++ b/src/mcp/service.js @@ -3,7 +3,7 @@ import { ValidationError, NotFoundError, GraphError } from '../errors.js'; import { ensureGitRepo, getFsmonitorStatus, hasGitRepo, lsRemote, pushWarpRefs } from '../git.js'; import { getLocalRepoDir, getThinkDir, getUpstreamUrl } from '../paths.js'; import { normalizeCaptureProvenance } from '../capture-provenance.js'; -import { getCaptureAmbientContext, getAmbientProjectContext } from '../project-context.js'; +import { getAmbientProjectContext } from '../project-context.js'; import { finalizeCapturedThought, getBrowseWindow, @@ -53,7 +53,6 @@ const defaultCaptureDeps = Object.freeze({ ensureGitRepo, finalizeCapturedThought, getAmbientProjectContext, - getCaptureAmbientContext, getCwd: () => process.cwd(), getGraphModelStatus, getLocalRepoDir, @@ -77,7 +76,8 @@ export function createCaptureThoughtService(deps = defaultCaptureDeps) { await deps.ensureGitRepo(repoDir); const entry = await deps.saveRawCapture(repoDir, thought, { provenance: captureProvenance, - ambientContext: deps.getCaptureAmbientContext(deps.getCwd()), + ambientContext: deps.getAmbientProjectContext(deps.getCwd()), + initializeGraphModel: !repoAlreadyExists, }); const followthrough = await runCaptureFollowthrough(deps, repoDir, entry.id, repoAlreadyExists); const backupStatus = await runCaptureBackup(deps, repoDir); diff --git a/src/minds.js b/src/minds.js index 49e33db8..6dd7b60a 100644 --- a/src/minds.js +++ b/src/minds.js @@ -29,7 +29,7 @@ export function discoverMinds(thinkDir = getThinkDir()) { continue; } - const isDefault = entry === 'repo'; + const isDefault = entry === 'james'; minds.push({ name: isDefault ? 'default' : entry, repoDir: fullPath, diff --git a/src/paths.js b/src/paths.js index 71a4c6a3..107d010b 100644 --- a/src/paths.js +++ b/src/paths.js @@ -15,7 +15,7 @@ export function getLocalRepoDir() { return path.resolve(configuredRepoDir); } - return path.join(getThinkDir(), 'repo'); + return path.join(getThinkDir(), 'james'); } export function getPromptMetricsFile() { diff --git a/src/store/annotate.js b/src/store/annotate.js index 856064db..5eb92b0b 100644 --- a/src/store/annotate.js +++ b/src/store/annotate.js @@ -1,26 +1,24 @@ import { randomUUID } from 'node:crypto'; -import { ValidationError, NotFoundError } from '../errors.js'; -import { ANNOTATION_PREFIX, TEXT_MIME } from './constants.js'; -import { encodeTextContent } from './content.js'; +import { NotFoundError, ValidationError } from '../errors.js'; +import { ANNOTATION_PREFIX } from './constants.js'; import { getCurrentTime } from './model.js'; +import { appendIndexedMemoryObject } from './native-index.js'; +import { openNativeMemory } from './native-runtime.js'; import { - commitThinkWorldline, getStoredEntry, openProductReadHandle, - openThinkWorldline, } from './runtime.js'; -export async function saveAnnotation(repoDir, targetEntryId, text, { writerId = null } = {}) { +export async function saveAnnotation(repoDir, targetEntryId, text, { + writerId = null, +} = {}) { if (!text || typeof text !== 'string' || text.trim() === '') { throw new ValidationError('Annotation text cannot be empty'); } - - const worldline = await openThinkWorldline(repoDir); + const memory = await openNativeMemory(repoDir); const read = await openProductReadHandle(repoDir); - const targetEntry = await getStoredEntry(read, targetEntryId); - - if (!targetEntry) { + if (!await getStoredEntry(read, targetEntryId)) { throw new NotFoundError(`Entry not found: ${targetEntryId}`); } @@ -29,26 +27,21 @@ export async function saveAnnotation(repoDir, targetEntryId, text, { writerId = const createdAt = timestamp.toISOString(); const sortKey = `${String(timestamp.getTime()).padStart(13, '0')}-${unique}`; const annotationId = `${ANNOTATION_PREFIX}${sortKey}`; - const resolvedWriterId = writerId ?? worldline.writerId; - await commitThinkWorldline(repoDir, async (patch) => { - patch - .addNode(annotationId) - .setProperty(annotationId, 'kind', 'annotation') - .setProperty(annotationId, 'source', 'annotation') - .setProperty(annotationId, 'channel', 'cli') - .setProperty(annotationId, 'writerId', resolvedWriterId) - .setProperty(annotationId, 'createdAt', createdAt) - .setProperty(annotationId, 'sortKey', sortKey) - .setProperty(annotationId, 'targetEntryId', targetEntryId) - .addEdge(annotationId, targetEntryId, 'annotates'); - - await patch.attachContent(annotationId, encodeTextContent(text.trim()), { mime: TEXT_MIME }); + await appendIndexedMemoryObject(repoDir, { + id: annotationId, + kind: 'annotation', + facts: { + kind: 'annotation', + text: text.trim(), + source: 'annotation', + channel: 'cli', + writerId: writerId ?? memory.writerId, + createdAt, + sortKey, + targetEntryId, + }, }); - return Object.freeze({ - annotationId, - targetEntryId, - createdAt, - }); + return Object.freeze({ annotationId, targetEntryId, createdAt }); } diff --git a/src/store/capture.js b/src/store/capture.js index 745d732a..6398b68e 100644 --- a/src/store/capture.js +++ b/src/store/capture.js @@ -1,150 +1,125 @@ import { normalizeCaptureProvenance } from '../capture-provenance.js'; -import { GRAPH_META_ID, GRAPH_MODEL_VERSION, TEXT_MIME } from './constants.js'; -import { encodeTextContent } from './content.js'; -import { createEntry } from './model.js'; import { - commitThinkWorldline, + GRAPH_META_ID, + GRAPH_MODEL_VERSION, +} from './constants.js'; +import { + createEntry, + createThoughtId, +} from './model.js'; +import { appendIndexedMemoryObject } from './native-index.js'; +import { + encodeNativeDocument, +} from './native-document.js'; +import { openNativeMemory } from './native-runtime.js'; +import { + getGraphModelStatusForRead, getStoredEntry, - openThinkWorldline, openProductReadHandle, } from './runtime.js'; -import { ensureCaptureReadEdges, ensureFirstDerivedArtifacts } from './derivation.js'; -import { migrateGraphModel } from './migrations.js'; -import { - applyCaptureReadModelPatch, - applyPendingCaptureReadModelPatch, -} from './read-model.js'; export async function saveRawCapture(repoDir, thought, { provenance = null, ambientContext = null, + initializeGraphModel = false, } = {}) { - return await writeRawCapture(repoDir, thought, { - provenance, - ambientContext, + const memory = await openNativeMemory(repoDir); + const created = createEntry( + thought, + memory.writerId, + { kind: 'capture', source: 'capture' } + ); + const captureProvenance = normalizeCaptureProvenance(provenance); + const thoughtId = createThoughtId(thought); + const entry = Object.freeze({ + ...created, + ...captureAmbientFacts(ambientContext), + ...captureProvenanceFacts(captureProvenance), + thoughtId, + sessionId: null, }); -} -async function writeRawCapture(repoDir, thought, { - provenance, - ambientContext, -}) { - const worldline = await openThinkWorldline(repoDir); - const entry = createEntry(thought, worldline.writerId, { kind: 'capture', source: 'capture' }); - const captureProvenance = normalizeCaptureProvenance(provenance); - const patcher = createRawCapturePatcher(entry, thought, { - ambientContext, - captureProvenance, + await appendIndexedMemoryObject(repoDir, { + id: entry.id, + kind: entry.kind, + facts: captureFacts(entry), }); - await commitThinkWorldline(repoDir, patcher); + if (initializeGraphModel) { + await writeGraphMetadata(memory, entry.createdAt); + } return entry; } -function createRawCapturePatcher(entry, thought, { ambientContext, captureProvenance }) { - return async (patch) => { - applyRawCapturePatch(patch, entry, { - ambientContext, - captureProvenance, - }); - await patch.attachContent(entry.id, encodeTextContent(thought), { mime: TEXT_MIME }); - }; +export async function finalizeCapturedThought(repoDir, entryId) { + const read = await openProductReadHandle(repoDir); + const entry = await getStoredEntry(read, entryId); + return Object.freeze({ + entry: entry?.kind === 'capture' ? entry : null, + migration: null, + }); } -function applyRawCapturePatch(patch, entry, { ambientContext, captureProvenance }) { - applyPendingCaptureReadModelPatch(patch, entry, { ambientContext }); - patch - .addNode(entry.id) - .setProperty(entry.id, 'kind', entry.kind) - .setProperty(entry.id, 'source', entry.source) - .setProperty(entry.id, 'channel', entry.channel) - .setProperty(entry.id, 'writerId', entry.writerId) - .setProperty(entry.id, 'createdAt', entry.createdAt) - .setProperty(entry.id, 'sortKey', entry.sortKey); - - applyAmbientContextPatch(patch, entry.id, ambientContext); - applyCaptureProvenancePatch(patch, entry.id, captureProvenance); +export async function getGraphModelStatus(repoDir) { + const read = await openProductReadHandle(repoDir); + return await getGraphModelStatusForRead(read); } -function applyCaptureProvenancePatch(patch, entryId, captureProvenance) { - if (captureProvenance?.ingress) { - patch.setProperty(entryId, 'captureIngress', captureProvenance.ingress); - } - if (captureProvenance?.sourceApp) { - patch.setProperty(entryId, 'captureSourceApp', captureProvenance.sourceApp); - } - if (captureProvenance?.sourceURL) { - patch.setProperty(entryId, 'captureSourceURL', captureProvenance.sourceURL); - } +function captureFacts(entry) { + return Object.freeze({ + kind: entry.kind, + text: entry.text, + source: entry.source, + channel: entry.channel, + writerId: entry.writerId, + createdAt: entry.createdAt, + sortKey: entry.sortKey, + thoughtId: entry.thoughtId, + sessionId: entry.sessionId, + ambientCwd: entry.ambientCwd, + ambientGitRoot: entry.ambientGitRoot, + ambientGitRemote: entry.ambientGitRemote, + ambientGitBranch: entry.ambientGitBranch, + captureIngress: entry.captureIngress, + captureSourceApp: entry.captureSourceApp, + captureSourceURL: entry.captureSourceURL, + }); } -export async function finalizeCapturedThought(repoDir, entryId, { - migrateIfNeeded = false, - ambientContext = null, -} = {}) { - let read = await openProductReadHandle(repoDir); - let entry = await getStoredEntry(read, entryId); - - if (!entry || entry.kind !== 'capture') { - return { - entry: null, - migration: null, - }; - } - - await patchCaptureReadModel(repoDir, entry, ambientContext); - read = await openProductReadHandle(repoDir); - entry = await getStoredEntry(read, entryId); - - await ensureFirstDerivedArtifacts(repoDir, read, entry); - read = await openProductReadHandle(repoDir); - await ensureCaptureReadEdges(repoDir, read, entryId); - read = await openProductReadHandle(repoDir); - entry = await getStoredEntry(read, entryId); - - return { - entry, - migration: migrateIfNeeded ? await migrateGraphModel(repoDir) : null, - }; +function captureAmbientFacts(context) { + const resolved = context ?? {}; + return Object.freeze({ + ambientCwd: nullable(resolved.cwd), + ambientGitRoot: nullable(resolved.gitRoot), + ambientGitRemote: nullable(resolved.gitRemote), + ambientGitBranch: nullable(resolved.gitBranch), + }); } -export async function getGraphModelStatus(repoDir) { - const worldline = await openThinkWorldline(repoDir); - const graphMeta = await worldline.live().getNodeProps(GRAPH_META_ID); - const currentGraphModelVersion = Number(graphMeta?.graphModelVersion ?? 1); - return { - currentGraphModelVersion, - requiredGraphModelVersion: GRAPH_MODEL_VERSION, - migrationRequired: currentGraphModelVersion < GRAPH_MODEL_VERSION, - }; +function captureProvenanceFacts(provenance) { + const resolved = provenance ?? {}; + return Object.freeze({ + captureIngress: nullable(resolved.ingress), + captureSourceApp: nullable(resolved.sourceApp), + captureSourceURL: nullable(resolved.sourceURL), + }); } -function applyAmbientContextPatch(patch, entryId, ambientContext) { - if (!ambientContext) { - return; - } - - if (ambientContext.cwd) { - patch.setProperty(entryId, 'ambientCwd', ambientContext.cwd); - } - if (ambientContext.gitRoot) { - patch.setProperty(entryId, 'ambientGitRoot', ambientContext.gitRoot); - } - if (ambientContext.gitRemote) { - patch.setProperty(entryId, 'ambientGitRemote', ambientContext.gitRemote); - } - if (ambientContext.gitBranch) { - patch.setProperty(entryId, 'ambientGitBranch', ambientContext.gitBranch); - } +function nullable(value) { + return value ?? null; } -async function patchCaptureReadModel(repoDir, entry, ambientContext) { - const patcher = async (patch) => { - const read = await openProductReadHandle(repoDir); - await applyCaptureReadModelPatch(patch, read, entry, { ambientContext }); - applyAmbientContextPatch(patch, entry.id, ambientContext); - }; - - await commitThinkWorldline(repoDir, patcher); +async function writeGraphMetadata(memory, timestamp) { + const document = Object.freeze({ + id: GRAPH_META_ID, + kind: 'graph_meta', + createdAt: timestamp, + graphModelVersion: GRAPH_MODEL_VERSION, + updatedAt: timestamp, + }); + await memory.writeMemoryDocument({ + id: GRAPH_META_ID, + bytes: encodeNativeDocument('memory-object', document), + }); } diff --git a/src/store/derivation.js b/src/store/derivation.js index 2c53c607..c40a6f34 100644 --- a/src/store/derivation.js +++ b/src/store/derivation.js @@ -1,16 +1,12 @@ -import { parseJson, stringifyJson } from '../json.js'; import { DERIVER_NAME, DERIVER_VERSION, - GRAPH_META_ID, - GRAPH_MODEL_VERSION, REFLECT_MARKERS, REFLECT_PROMPT_TYPES, SCHEMA_VERSION, SESSION_IDLE_GAP_MS, SESSION_PREFIX, } from './constants.js'; -import { encodeTextContent } from './content.js'; import { compareEntriesNewestFirst, createArtifactId, @@ -19,17 +15,11 @@ import { normalizeSeed, } from './model.js'; import { - commitThinkWorldline, - getLatestStoredEntry, - getProducedInSessionId, - getStoredEntry, - hasNode, listEntriesByKind, } from './runtime.js'; export function assessReflectability(text) { const seedQuality = deriveSeedQuality(createThoughtId(text), text); - if (seedQuality.verdict === 'likely_reflectable') { return Object.freeze({ eligible: true, @@ -37,7 +27,6 @@ export function assessReflectability(text) { text: 'This entry looks like a candidate idea, question, or decision that can be pressure-tested.', }); } - return Object.freeze({ eligible: false, kind: 'not_pressure_testable', @@ -46,174 +35,24 @@ export function assessReflectability(text) { }); } -export async function ensureFirstDerivedArtifacts(repoDir, read, entry) { - if (!entry || entry.kind !== 'capture') { - return null; - } - - const thoughtId = createThoughtId(entry.text); - const seedQuality = deriveSeedQuality(thoughtId, entry.text); - const sessionAttribution = await deriveSessionAttribution(read, entry); - - const [ - thoughtNodeExists, - seedQualityExists, - sessionNodeExists, - sessionArtifactExists, - entryProps, - graphMetaProps, - ] = await Promise.all([ - hasNode(read, thoughtId), - hasNode(read, seedQuality.artifactId), - hasNode(read, sessionAttribution.sessionId), - hasNode(read, sessionAttribution.artifactId), - read.view.getNodeProps(entry.id), - read.view.getNodeProps(GRAPH_META_ID), - ]); - - const needsCaptureThoughtLink = entryProps?.thoughtId !== thoughtId; - const needsCaptureSessionLink = entryProps?.sessionId !== sessionAttribution.sessionId; - const needsGraphMetadata = !graphMetaProps || graphMetaProps.graphModelVersion !== GRAPH_MODEL_VERSION; - - if ( - thoughtNodeExists - && seedQualityExists - && sessionNodeExists - && sessionArtifactExists - && !needsCaptureThoughtLink - && !needsCaptureSessionLink - && !needsGraphMetadata - ) { - return { - thoughtId, - seedQuality, - sessionAttribution, - }; - } - - await commitThinkWorldline(repoDir, async (patch) => { - ensureGraphMetadataNode(patch, graphMetaProps); - - if (!thoughtNodeExists) { - patch - .addNode(thoughtId) - .setProperty(thoughtId, 'kind', 'thought') - .setProperty(thoughtId, 'fingerprint', thoughtId.slice('thought:'.length)) - .setProperty(thoughtId, 'createdAt', entry.createdAt) - .setProperty(thoughtId, 'schemaVersion', SCHEMA_VERSION); - - await patch.attachContent(thoughtId, encodeTextContent(entry.text), { mime: 'text/plain; charset=utf-8' }); - } - - if (needsCaptureThoughtLink) { - patch.setProperty(entry.id, 'thoughtId', thoughtId); - } - patch.addEdge(entry.id, thoughtId, 'expresses'); - - if (!seedQualityExists) { - addArtifactNode(patch, seedQuality); - } - - if (!sessionNodeExists) { - patch - .addNode(sessionAttribution.sessionId) - .setProperty(sessionAttribution.sessionId, 'kind', 'session') - .setProperty(sessionAttribution.sessionId, 'createdAt', sessionAttribution.sessionCreatedAt) - .setProperty(sessionAttribution.sessionId, 'startSortKey', sessionAttribution.sessionStartSortKey) - .setProperty(sessionAttribution.sessionId, 'schemaVersion', SCHEMA_VERSION); - } - - if (needsCaptureSessionLink) { - patch.setProperty(entry.id, 'sessionId', sessionAttribution.sessionId); - } - patch.addEdge(entry.id, sessionAttribution.sessionId, 'captured_in'); - - if (!sessionArtifactExists) { - addArtifactNode(patch, sessionAttribution); - } - }); - - return { - thoughtId, - seedQuality, - sessionAttribution, - }; -} - -export async function ensureCaptureReadEdges(repoDir, read, entryId) { - const entry = await getStoredEntry(read, entryId); - if (!entry || entry.kind !== 'capture') { - return; - } - - const olderEntry = await getLatestStoredEntry(read, 'capture', { - excludeIds: [entry.id], - }); - - if (!olderEntry) { - return; - } - - await commitThinkWorldline(repoDir, (patch) => { - patch.addEdge(entry.id, olderEntry.id, 'older'); - patch.addEdge(olderEntry.id, entry.id, 'newer'); - }); -} - export async function listDirectDerivedReceipts(read, seedEntryId) { - const receipts = []; - const seenEntryIds = new Set(); - const graphNativeNeighbors = await read.view.query() - .match(seedEntryId) - .incoming('responds_to') - .run(); - - for (const neighbor of graphNativeNeighbors.nodes ?? []) { - // eslint-disable-next-line no-await-in-loop -- sequential graph traversal of neighbor nodes - const entry = await getStoredEntry(read, neighbor.id, neighbor.props ?? null); - if (!entry || entry.kind !== 'reflect' || seenEntryIds.has(entry.id)) { - continue; - } - - seenEntryIds.add(entry.id); - receipts.push({ - relation: 'seed_of', - kind: entry.kind, - entryId: entry.id, - // eslint-disable-next-line no-await-in-loop -- sequential graph read per neighbor - sessionId: await getProducedInSessionId(read, entry), - promptType: entry.promptType, - createdAt: entry.createdAt, - sortKey: entry.sortKey, - }); - } - const reflectEntries = await listEntriesByKind(read, 'reflect'); - for (const entry of reflectEntries) { - if (entry.seedEntryId !== seedEntryId || seenEntryIds.has(entry.id)) { - continue; - } - - receipts.push({ + return reflectEntries + .filter(entry => entry.seedEntryId === seedEntryId) + .sort(compareEntriesNewestFirst) + .map(entry => Object.freeze({ relation: 'seed_of', kind: entry.kind, entryId: entry.id, sessionId: entry.sessionId, promptType: entry.promptType, createdAt: entry.createdAt, - sortKey: entry.sortKey, - }); - } - - return receipts - .sort(compareEntriesNewestFirst) - .map(({ sortKey: _sortKey, ...receipt }) => receipt); + })); } export function deriveSeedQuality(thoughtId, text) { const normalized = normalizeSeed(text); - const eligible = REFLECT_MARKERS.some((pattern) => pattern.test(normalized)); - + const eligible = REFLECT_MARKERS.some(pattern => pattern.test(normalized)); return Object.freeze({ artifactId: createArtifactId('seed_quality', thoughtId), kind: 'seed_quality', @@ -233,192 +72,61 @@ export function deriveSeedQuality(thoughtId, text) { } export async function deriveSessionAttribution(read, entry) { - const latestEntry = await getLatestStoredEntry(read, 'capture', { - excludeIds: [entry.id], - }); - + const captures = await listEntriesByKind(read, 'capture', { limit: 4096 }); + const latestEntry = captures.find(candidate => + candidate.id !== entry.id + && Date.parse(candidate.createdAt) <= Date.parse(entry.createdAt) + ); if (latestEntry && latestEntry.id !== entry.id) { const gapMs = Date.parse(entry.createdAt) - Date.parse(latestEntry.createdAt); if (gapMs <= SESSION_IDLE_GAP_MS) { const activeSessionId = latestEntry.sessionId || `${SESSION_PREFIX}${latestEntry.sortKey}`; - const sessionCreatedAt = latestEntry.sessionCreatedAt || latestEntry.createdAt; - const sessionStartSortKey = latestEntry.sessionStartSortKey || latestEntry.sortKey; - - return Object.freeze({ - artifactId: createArtifactId('session_attribution', entry.id, activeSessionId), - kind: 'session_attribution', - primaryInputKind: 'capture', - primaryInputId: entry.id, - sessionId: activeSessionId, - sessionCreatedAt, - sessionStartSortKey, + return buildSessionAttribution(entry, activeSessionId, { + sessionCreatedAt: latestEntry.createdAt, + sessionStartSortKey: latestEntry.sortKey, reasonKind: 'temporal_proximity', reasonText: 'Captured within 5 minutes of the most recent entry.', - deriver: DERIVER_NAME, - deriverVersion: DERIVER_VERSION, - schemaVersion: SCHEMA_VERSION, - createdAt: getCurrentTime().toISOString(), }); } } - const sessionId = `${SESSION_PREFIX}${entry.sortKey}`; - return Object.freeze({ - artifactId: createArtifactId('session_attribution', entry.id, sessionId), - kind: 'session_attribution', - primaryInputKind: 'capture', - primaryInputId: entry.id, - sessionId, + return buildSessionAttribution(entry, sessionId, { sessionCreatedAt: entry.createdAt, sessionStartSortKey: entry.sortKey, reasonKind: 'new_session_bucket', reasonText: 'Started a new session bucket because no recent capture fell within the 5 minute idle-gap threshold.', - deriver: DERIVER_NAME, - deriverVersion: DERIVER_VERSION, - schemaVersion: SCHEMA_VERSION, - createdAt: getCurrentTime().toISOString(), }); } export async function getCanonicalThought(read, entry) { const thoughtId = entry.thoughtId ?? createThoughtId(entry.text); - const thoughtProps = await read.view.getNodeProps(thoughtId); - return Object.freeze({ entryId: entry.id, thoughtId, relation: 'expresses', - stored: Boolean(thoughtProps), + stored: await read.memory.exists(thoughtId), }); } -export async function getSeedQualityReceipt(read, entry) { - const thoughtId = entry.thoughtId ?? createThoughtId(entry.text); - const artifactId = createArtifactId('seed_quality', thoughtId); - const props = await read.view.getNodeProps(artifactId); - if (!props) { - return null; - } - - return Object.freeze({ - artifactId, - kind: 'seed_quality', - primaryInputKind: props.primaryInputKind, - primaryInputId: props.primaryInputId, - verdict: props.verdict, - reasonKind: props.reasonKind, - reasonText: props.reasonText, - promptFamilies: Object.freeze(parseJsonArray(props.promptFamiliesJson)), - deriver: props.deriver, - deriverVersion: props.deriverVersion, - schemaVersion: props.schemaVersion, - createdAt: props.createdAt, - }); +export function getSeedQualityReceipt(_read, entry) { + return deriveSeedQuality(entry.thoughtId ?? createThoughtId(entry.text), entry.text); } export async function getSessionAttributionReceipt(read, entry) { - const thoughtId = entry.sessionId ?? null; - const artifactId = thoughtId - ? createArtifactId('session_attribution', entry.id, thoughtId) - : (await deriveSessionAttribution(read, entry)).artifactId; - const props = await read.view.getNodeProps(artifactId); - if (!props) { - return null; - } - - return Object.freeze({ - artifactId, - kind: 'session_attribution', - primaryInputKind: props.primaryInputKind, - primaryInputId: props.primaryInputId, - sessionId: props.sessionId, - reasonKind: props.reasonKind, - reasonText: props.reasonText, - deriver: props.deriver, - deriverVersion: props.deriverVersion, - schemaVersion: props.schemaVersion, - createdAt: props.createdAt, - }); + return await deriveSessionAttribution(read, entry); } -export async function getSessionAttributionReceiptIfPresent(read, entry) { - if (!entry.sessionId) { - return null; - } - - const artifactId = createArtifactId('session_attribution', entry.id, entry.sessionId); - const props = await read.view.getNodeProps(artifactId); - if (!props) { - return null; - } - +function buildSessionAttribution(entry, sessionId, details) { return Object.freeze({ - artifactId, + artifactId: createArtifactId('session_attribution', entry.id, sessionId), kind: 'session_attribution', - primaryInputKind: props.primaryInputKind, - primaryInputId: props.primaryInputId, - sessionId: props.sessionId, - reasonKind: props.reasonKind, - reasonText: props.reasonText, - deriver: props.deriver, - deriverVersion: props.deriverVersion, - schemaVersion: props.schemaVersion, - createdAt: props.createdAt, + primaryInputKind: 'capture', + primaryInputId: entry.id, + sessionId, + ...details, + deriver: DERIVER_NAME, + deriverVersion: DERIVER_VERSION, + schemaVersion: SCHEMA_VERSION, + createdAt: getCurrentTime().toISOString(), }); } - -function addArtifactNode(patch, artifact) { - patch - .addNode(artifact.artifactId) - .setProperty(artifact.artifactId, 'kind', artifact.kind) - .setProperty(artifact.artifactId, 'primaryInputKind', artifact.primaryInputKind) - .setProperty(artifact.artifactId, 'primaryInputId', artifact.primaryInputId) - .setProperty(artifact.artifactId, 'deriver', artifact.deriver) - .setProperty(artifact.artifactId, 'deriverVersion', artifact.deriverVersion) - .setProperty(artifact.artifactId, 'schemaVersion', artifact.schemaVersion) - .setProperty(artifact.artifactId, 'createdAt', artifact.createdAt); - - if (artifact.kind === 'seed_quality') { - patch - .setProperty(artifact.artifactId, 'verdict', artifact.verdict) - .setProperty(artifact.artifactId, 'reasonKind', artifact.reasonKind) - .setProperty(artifact.artifactId, 'reasonText', artifact.reasonText) - .setProperty(artifact.artifactId, 'promptFamiliesJson', stringifyJson(artifact.promptFamilies)); - patch.addEdge(artifact.artifactId, artifact.primaryInputId, 'derived_from'); - return; - } - - if (artifact.kind === 'session_attribution') { - patch - .setProperty(artifact.artifactId, 'sessionId', artifact.sessionId) - .setProperty(artifact.artifactId, 'reasonKind', artifact.reasonKind) - .setProperty(artifact.artifactId, 'reasonText', artifact.reasonText); - patch.addEdge(artifact.artifactId, artifact.primaryInputId, 'contextualizes'); - } -} - -function ensureGraphMetadataNode(patch, graphMetaProps) { - if (!graphMetaProps) { - patch - .addNode(GRAPH_META_ID) - .setProperty(GRAPH_META_ID, 'kind', 'graph_meta') - .setProperty(GRAPH_META_ID, 'createdAt', getCurrentTime().toISOString()); - } - - patch - .setProperty(GRAPH_META_ID, 'graphModelVersion', GRAPH_MODEL_VERSION) - .setProperty(GRAPH_META_ID, 'updatedAt', getCurrentTime().toISOString()); -} - -function parseJsonArray(value) { - if (!value) { - return []; - } - - try { - const parsed = parseJson(value); - return Array.isArray(parsed) ? parsed : []; - } catch { - return []; - } -} diff --git a/src/store/enrichment/runner.js b/src/store/enrichment/runner.js index bef8bbfa..778560ba 100644 --- a/src/store/enrichment/runner.js +++ b/src/store/enrichment/runner.js @@ -1,8 +1,20 @@ -import { CLASSIFICATION_PREFIX, TOPIC_PREFIX, KEYWORD_PREFIX, GRAPH_META_ID } from '../constants.js'; +import { + CLASSIFICATION_PREFIX, + GRAPH_META_ID, + KEYWORD_PREFIX, + TOPIC_PREFIX, +} from '../constants.js'; import { createArtifactId, getCurrentTime } from '../model.js'; import { - commitThinkWorldline, - getStoredEntry, + decodeNativeDocument, + encodeNativeDocument, +} from '../native-document.js'; +import { + appendIndexedMemoryObject, + readIndexedMemoryBatch, +} from '../native-index.js'; +import { openNativeMemory } from '../native-runtime.js'; +import { listEntriesByKind, openProductReadHandle, } from '../runtime.js'; @@ -11,338 +23,220 @@ import { extractTopics } from './auto-tags.js'; import { classifyThought } from './semantic-parse.js'; const TOPIC_PROMOTION_THRESHOLD = 2; +const ENRICHMENT_BATCH_SIZE = 500; -/** - * Run the enrichment pipeline on all un-enriched captures in a repo. - * Uses worldline query API — no full graph materialization. - */ export async function runEnrichmentPipeline(repoDir) { - const read = await openProductReadHandle(repoDir); - const { view } = read; - - // 1. Determine the starting point (high-water mark cursor) - const metaProps = await view.getNodeProps(GRAPH_META_ID); - const cursorId = metaProps?.lastEnrichedCaptureId; - - let captures = []; - if (cursorId && await view.hasNode(cursorId)) { - // Incremental path: Traverse 'newer' edges from the cursor - const forwardIds = await view.traverse.bfs(cursorId, { - dir: 'out', - labelFilter: 'newer', - }); - - for (const id of forwardIds) { - if (id === cursorId) { continue; } - // eslint-disable-next-line no-await-in-loop -- sequential retrieval of new captures - const entry = await getStoredEntry(read, id); - if (entry && entry.kind === 'capture') { - captures.push(entry); - } - } - } else { - // Bootstrap path: O(N) scan (only happens once or if cursor is lost) - captures = await listEntriesByKind(read, 'capture'); - } - + const memory = await openNativeMemory(repoDir); + const graphMetadata = decodeNativeDocument( + await memory.memoryDocument(GRAPH_META_ID), + 'memory-object' + ); + const batch = await readIndexedMemoryBatch(repoDir, 'capture', { + after: graphMetadata?.lastEnrichedCaptureId + ? { + id: graphMetadata.lastEnrichedCaptureId, + pageNumber: graphMetadata.lastEnrichedCapturePage, + offset: graphMetadata.lastEnrichedCaptureOffset, + } + : null, + limit: ENRICHMENT_BATCH_SIZE, + }); + const captures = batch.documents; if (captures.length === 0) { - return Object.freeze({ - capturesProcessed: 0, - topicNodesCreated: 0, - keywordNodesCreated: 0, - aboutEdgesAdded: 0, - mentionsEdgesAdded: 0, - classifiedEdgesAdded: 0, - receiptsCreated: 0, - promotedTopics: [], - }); - } - - // Find existing auto_tags receipts via query - const existingReceipts = new Set(); - const tagReceiptResult = await view.query().match('artifact:*').where({ kind: 'auto_tags' }).run(); - for (const node of tagReceiptResult.nodes ?? []) { - if (node.props.primaryInputId) { - existingReceipts.add(node.props.primaryInputId); - } - } - - // Find existing semantic_parse receipts via query - const existingParseReceipts = new Set(); - const parseReceiptResult = await view.query().match('artifact:*').where({ kind: 'semantic_parse' }).run(); - for (const node of parseReceiptResult.nodes ?? []) { - if (node.props.primaryInputId) { - existingParseReceipts.add(node.props.primaryInputId); - } - } - - // Find existing topic nodes via query - const existingTopicNodes = new Set(); - const topicResult = await view.query().match(`${TOPIC_PREFIX}*`).run(); - for (const node of topicResult.nodes ?? []) { - existingTopicNodes.add(node.id); - } - - // Find existing keyword nodes via query - const existingKeywordNodes = new Set(); - const keywordResult = await view.query().match(`${KEYWORD_PREFIX}*`).run(); - for (const node of keywordResult.nodes ?? []) { - existingKeywordNodes.add(node.id); + return emptyResult(); } + const read = await openProductReadHandle(repoDir); + const keywordIds = await indexedIds(read, 'keyword'); + const topicIds = await indexedIds(read, 'topic'); + const autoTagIds = await indexedIds(read, 'auto_tags'); + const semanticParseIds = await indexedIds(read, 'semantic_parse'); - // Track candidate topic counts and classifications across all captures + const timestamp = getCurrentTime().toISOString(); const topicCounts = new Map(); - const thoughtTopics = new Map(); - const thoughtClassifications = new Map(); + const captureTopics = new Map(); + const captureClassifications = new Map(); for (const capture of captures) { - const { thoughtId } = capture; - if (!thoughtId) { continue; } - - if (!thoughtTopics.has(thoughtId)) { - const topics = extractTopics(capture.text); - thoughtTopics.set(thoughtId, topics); - for (const topic of topics) { - topicCounts.set(topic, (topicCounts.get(topic) || 0) + 1); - } - } - - if (!thoughtClassifications.has(thoughtId)) { - thoughtClassifications.set(thoughtId, classifyThought(capture.text)); - } - } - - // Determine promoted topics - const promotedTopics = new Set(); - for (const [topic, count] of topicCounts) { - if (count >= TOPIC_PROMOTION_THRESHOLD) { - promotedTopics.add(topic); - } - } - - // Check existing about edges per thought via traversal - const existingAboutEdges = new Set(); - for (const [thoughtId] of thoughtTopics) { - // eslint-disable-next-line no-await-in-loop -- per-thought traversal - const traversal = await view.query().match(thoughtId).outgoing('about').run(); - for (const node of traversal.nodes ?? []) { - existingAboutEdges.add(`${thoughtId}\0${node.id}`); - } - } - - // Check existing mentions edges per thought via traversal (inverted index) - const existingMentionsEdges = new Set(); - for (const [thoughtId] of thoughtTopics) { - // eslint-disable-next-line no-await-in-loop -- per-thought traversal - const traversal = await view.query().match(thoughtId).outgoing('mentions').run(); - for (const node of traversal.nodes ?? []) { - existingMentionsEdges.add(`${thoughtId}\0${node.id}`); - } - } - - // Check existing classified_as edges per thought via traversal - const existingClassifiedEdges = new Set(); - for (const [thoughtId] of thoughtClassifications) { - // eslint-disable-next-line no-await-in-loop -- per-thought traversal - const traversal = await view.query().match(thoughtId).outgoing('classified_as').run(); - for (const node of traversal.nodes ?? []) { - existingClassifiedEdges.add(`${thoughtId}\0${node.id}`); + const topics = extractTopics(capture.text); + captureTopics.set(capture.id, topics); + captureClassifications.set(capture.id, classifyThought(capture.text)); + for (const topic of topics) { + topicCounts.set(topic, (topicCounts.get(topic) ?? 0) + 1); } } - const timestamp = getCurrentTime().toISOString(); - const keywordNodesToCreate = []; - const mentionsEdgesToAdd = []; - const topicNodesToCreate = []; - const aboutEdgesToAdd = []; - const autoTagReceiptsToCreate = []; - const classifiedEdgesToAdd = []; - const semanticParseReceiptsToCreate = []; + const promotedTopics = [...topicCounts] + .filter(([, count]) => count >= TOPIC_PROMOTION_THRESHOLD) + .map(([topic]) => topic) + .sort(); + let keywordNodesCreated = 0; + let topicNodesCreated = 0; + let mentionsEdgesAdded = 0; + let aboutEdgesAdded = 0; + let classifiedEdgesAdded = 0; + let receiptsCreated = 0; - for (const [thoughtId, topics] of thoughtTopics) { + for (const capture of captures) { + const topics = captureTopics.get(capture.id) ?? []; for (const keyword of topics) { - const keywordNodeId = `${KEYWORD_PREFIX}${keyword}`; - if (!existingKeywordNodes.has(keywordNodeId)) { - keywordNodesToCreate.push({ keywordNodeId, keyword }); - existingKeywordNodes.add(keywordNodeId); + const keywordId = `${KEYWORD_PREFIX}${keyword}`; + if (!keywordIds.has(keywordId)) { + // eslint-disable-next-line no-await-in-loop -- enrichment admits bounded domain Intents deliberately + await appendIndexedMemoryObject(repoDir, { + id: keywordId, + kind: 'keyword', + facts: { kind: 'keyword', name: keyword, createdAt: timestamp }, + }); + keywordIds.add(keywordId); + keywordNodesCreated += 1; } - - const edgeKey = `${thoughtId}\0${keywordNodeId}`; - if (!existingMentionsEdges.has(edgeKey)) { - mentionsEdgesToAdd.push({ thoughtId, keywordNodeId }); - existingMentionsEdges.add(edgeKey); + mentionsEdgesAdded += 1; + } + + for (const topic of topics.filter(candidate => promotedTopics.includes(candidate))) { + const topicId = `${TOPIC_PREFIX}${topic}`; + if (!topicIds.has(topicId)) { + // eslint-disable-next-line no-await-in-loop -- enrichment admits bounded domain Intents deliberately + await appendIndexedMemoryObject(repoDir, { + id: topicId, + kind: 'topic', + facts: { + kind: 'topic', + name: topic, + normalizedName: topic, + createdAt: timestamp, + source: 'auto_tags', + thoughtCount: topicCounts.get(topic) ?? 0, + }, + }); + topicIds.add(topicId); + topicNodesCreated += 1; } - } - } - - for (const topic of promotedTopics) { - const nodeId = `${TOPIC_PREFIX}${topic}`; - if (!existingTopicNodes.has(nodeId)) { - topicNodesToCreate.push({ nodeId, topic }); - existingTopicNodes.add(nodeId); - } - } - - for (const [thoughtId, topics] of thoughtTopics) { - for (const topic of topics) { - if (!promotedTopics.has(topic)) { continue; } - const topicNodeId = `${TOPIC_PREFIX}${topic}`; - const edgeKey = `${thoughtId}\0${topicNodeId}`; - if (!existingAboutEdges.has(edgeKey)) { - aboutEdgesToAdd.push({ thoughtId, topicNodeId }); - existingAboutEdges.add(edgeKey); + aboutEdgesAdded += 1; + } + + const classification = captureClassifications.get(capture.id); + for (const name of classification?.classifications ?? []) { + const classificationId = `${CLASSIFICATION_PREFIX}${name}`; + // eslint-disable-next-line no-await-in-loop -- enrichment declares bounded ontology endpoints + if (!await memory.exists(classificationId)) { + // eslint-disable-next-line no-await-in-loop -- enrichment admits bounded domain Intents deliberately + await memory.writeMemoryDocument({ + id: classificationId, + bytes: encodeNativeDocument('memory-object', { + id: classificationId, + kind: 'classification', + name, + createdAt: timestamp, + }), + }); } - } - } - - for (const capture of captures) { - const { thoughtId } = capture; - if (!thoughtId || existingReceipts.has(thoughtId)) { continue; } - - autoTagReceiptsToCreate.push({ - artifactId: createArtifactId('auto_tags', thoughtId), - thoughtId, - topics: thoughtTopics.get(thoughtId) || [], + classifiedEdgesAdded += 1; + } + + const tagArtifactId = createArtifactId('auto_tags', capture.id); + if (!autoTagIds.has(tagArtifactId)) { + // eslint-disable-next-line no-await-in-loop -- receipt objects use the bounded native document index + await appendIndexedMemoryObject(repoDir, { + id: tagArtifactId, + kind: 'auto_tags', + facts: { + kind: 'auto_tags', + primaryInputKind: 'capture', + primaryInputId: capture.id, + topicsExtractedCount: topics.length, + method: 'keyword-extraction', + deriver: 'think', + deriverVersion: '1', + schemaVersion: '1', + createdAt: timestamp, + }, + }); + autoTagIds.add(tagArtifactId); + receiptsCreated += 1; + } + + const parseArtifactId = createArtifactId('semantic_parse', capture.id); + if (!semanticParseIds.has(parseArtifactId)) { + // eslint-disable-next-line no-await-in-loop -- receipt objects use the bounded native document index + await appendIndexedMemoryObject(repoDir, { + id: parseArtifactId, + kind: 'semantic_parse', + facts: { + kind: 'semantic_parse', + primaryInputKind: 'capture', + primaryInputId: capture.id, + classificationCount: classification?.classifications?.length ?? 0, + markerCount: classification?.markers?.length ?? 0, + deriver: 'think', + deriverVersion: '1', + schemaVersion: '1', + createdAt: timestamp, + }, + }); + semanticParseIds.add(parseArtifactId); + receiptsCreated += 1; + } + } + + const latest = captures.at(-1); + if (latest && batch.cursor) { + await memory.writeMemoryDocument({ + id: GRAPH_META_ID, + bytes: encodeNativeDocument('memory-object', { + ...graphMetadata, + id: GRAPH_META_ID, + kind: 'graph_meta', + lastEnrichedCaptureId: latest.id, + lastEnrichedCapturePage: batch.cursor.pageNumber, + lastEnrichedCaptureOffset: batch.cursor.offset, + updatedAt: timestamp, + }), + declare: graphMetadata === null, }); - existingReceipts.add(thoughtId); } - - for (const [thoughtId, result] of thoughtClassifications) { - for (const classification of result.classifications) { - const classNodeId = `${CLASSIFICATION_PREFIX}${classification}`; - const edgeKey = `${thoughtId}\0${classNodeId}`; - if (!existingClassifiedEdges.has(edgeKey)) { - classifiedEdgesToAdd.push({ thoughtId, classNodeId }); - existingClassifiedEdges.add(edgeKey); - } - } - } - - for (const capture of captures) { - const { thoughtId } = capture; - if (!thoughtId || existingParseReceipts.has(thoughtId)) { continue; } - - const result = thoughtClassifications.get(thoughtId); - if (!result) { continue; } - - semanticParseReceiptsToCreate.push({ - artifactId: createArtifactId('semantic_parse', thoughtId), - thoughtId, - result, - }); - existingParseReceipts.add(thoughtId); - } - - await commitThinkWorldline(repoDir, (patch) => { - // Create keyword nodes and mentions edges (The Inverted Index) - for (const { keywordNodeId, keyword } of keywordNodesToCreate) { - patch - .addNode(keywordNodeId) - .setProperty(keywordNodeId, 'kind', 'keyword') - .setProperty(keywordNodeId, 'name', keyword) - .setProperty(keywordNodeId, 'createdAt', timestamp); - } - - for (const { thoughtId, keywordNodeId } of mentionsEdgesToAdd) { - patch.addEdge(thoughtId, keywordNodeId, 'mentions'); - } - - // Create promoted topic nodes - for (const { nodeId, topic } of topicNodesToCreate) { - patch - .addNode(nodeId) - .setProperty(nodeId, 'kind', 'topic') - .setProperty(nodeId, 'name', topic) - .setProperty(nodeId, 'normalizedName', topic) - .setProperty(nodeId, 'createdAt', timestamp) - .setProperty(nodeId, 'source', 'auto_tags'); - } - - // Add about edges for promoted topics - for (const { thoughtId, topicNodeId } of aboutEdgesToAdd) { - patch.addEdge(thoughtId, topicNodeId, 'about'); - } - - // Create auto_tags receipt artifacts - for (const { artifactId, thoughtId, topics } of autoTagReceiptsToCreate) { - patch - .addNode(artifactId) - .setProperty(artifactId, 'kind', 'auto_tags') - .setProperty(artifactId, 'primaryInputKind', 'thought') - .setProperty(artifactId, 'primaryInputId', thoughtId) - .setProperty(artifactId, 'topicsExtracted', JSON.stringify(topics)) - .setProperty(artifactId, 'method', 'keyword-extraction') - .setProperty(artifactId, 'topicNodesCreated', 0) - .setProperty(artifactId, 'deriver', 'think') - .setProperty(artifactId, 'deriverVersion', '1') - .setProperty(artifactId, 'schemaVersion', '1') - .setProperty(artifactId, 'createdAt', timestamp) - .addEdge(artifactId, thoughtId, 'derived_from'); - } - - // Add classified_as edges - for (const { thoughtId, classNodeId } of classifiedEdgesToAdd) { - patch.addEdge(thoughtId, classNodeId, 'classified_as'); - } - - // Create semantic_parse receipt artifacts - for (const { artifactId, thoughtId, result } of semanticParseReceiptsToCreate) { - patch - .addNode(artifactId) - .setProperty(artifactId, 'kind', 'semantic_parse') - .setProperty(artifactId, 'primaryInputKind', 'thought') - .setProperty(artifactId, 'primaryInputId', thoughtId) - .setProperty(artifactId, 'classifications', JSON.stringify(result.classifications)) - .setProperty(artifactId, 'markers', JSON.stringify(result.markers)) - .setProperty(artifactId, 'deriver', 'think') - .setProperty(artifactId, 'deriverVersion', '1') - .setProperty(artifactId, 'schemaVersion', '1') - .setProperty(artifactId, 'createdAt', timestamp) - .addEdge(artifactId, thoughtId, 'derived_from'); - } - - // Update the high-water mark cursor to the latest capture processed - const latestProcessed = [...captures].sort((a, b) => b.createdAt.localeCompare(a.createdAt))[0]; - if (latestProcessed) { - patch.setProperty(GRAPH_META_ID, 'lastEnrichedCaptureId', latestProcessed.id); - } - }); - invalidateSearchIndex(repoDir); return Object.freeze({ capturesProcessed: captures.length, - topicNodesCreated: topicNodesToCreate.length, - keywordNodesCreated: keywordNodesToCreate.length, - aboutEdgesAdded: aboutEdgesToAdd.length, - mentionsEdgesAdded: mentionsEdgesToAdd.length, - classifiedEdgesAdded: classifiedEdgesToAdd.length, - receiptsCreated: autoTagReceiptsToCreate.length + semanticParseReceiptsToCreate.length, - promotedTopics: [...promotedTopics].sort(), + topicNodesCreated, + keywordNodesCreated, + aboutEdgesAdded, + mentionsEdgesAdded, + classifiedEdgesAdded, + receiptsCreated, + promotedTopics, }); } -/** - * List all promoted topics in the graph with thought counts. - * Uses worldline query API — no full graph materialization. - */ export async function listTopics(repoDir) { const read = await openProductReadHandle(repoDir); + const topics = await listEntriesByKind(read, 'topic'); + return topics + .map(topic => Object.freeze({ + name: topic.name ?? topic.text, + thoughtCount: Number(topic.thoughtCount ?? 0), + createdAt: topic.createdAt, + })) + .sort((left, right) => right.thoughtCount - left.thoughtCount); +} - const topicResult = await read.view.query().match(`${TOPIC_PREFIX}*`).where({ kind: 'topic' }).run(); - const topics = []; - - for (const node of topicResult.nodes ?? []) { - // eslint-disable-next-line no-await-in-loop -- per-topic traversal for count - const incoming = await read.view.query().match(node.id).incoming('about').run(); - const thoughtCount = (incoming.nodes ?? []).length; - - topics.push(Object.freeze({ - name: node.props.name, - thoughtCount, - createdAt: node.props.createdAt, - })); - } +async function indexedIds(read, kind) { + return new Set( + (await listEntriesByKind(read, kind, { limit: 4096 })) + .map(entry => entry.id) + ); +} - return topics.sort((a, b) => b.thoughtCount - a.thoughtCount); +function emptyResult() { + return Object.freeze({ + capturesProcessed: 0, + topicNodesCreated: 0, + keywordNodesCreated: 0, + aboutEdgesAdded: 0, + mentionsEdgesAdded: 0, + classifiedEdgesAdded: 0, + receiptsCreated: 0, + promotedTopics: [], + }); } diff --git a/src/store/migrations.js b/src/store/migrations.js index 7c4982f5..240ca130 100644 --- a/src/store/migrations.js +++ b/src/store/migrations.js @@ -1,134 +1,30 @@ import { - ARTIFACT_PREFIX, CLASSIFICATION_PREFIX, CLASSIFICATIONS, GRAPH_META_ID, GRAPH_MODEL_VERSION, } from './constants.js'; -import { compareEntriesNewestFirst, getCurrentTime } from './model.js'; -import { commitThinkWorldline, commitThinkWorldlineWithWriter, openThinkWorldline } from './runtime.js'; +import { getCurrentTime } from './model.js'; +import { + decodeNativeDocument, + encodeNativeDocument, +} from './native-document.js'; +import { ensureEmptyNativeIndex } from './native-index.js'; +import { openNativeMemory } from './native-runtime.js'; export async function migrateGraphModel(repoDir) { - const worldline = await openThinkWorldline(repoDir); - const read = worldline.live(); - - // Check current History model version - const graphMeta = await read.getNodeProps(GRAPH_META_ID); - const needsMetadataNode = !graphMeta; - const needsGraphVersionUpdate = !graphMeta || graphMeta.graphModelVersion !== GRAPH_MODEL_VERSION; - - // Query each node kind separately — no full materialization - const captureResult = await read.query().match('entry:*').where({ kind: 'capture' }).run(); - const reflectEntryResult = await read.query().match('entry:*').where({ kind: 'reflect' }).run(); - const brainstormEntryResult = await read.query().match('entry:*').where({ kind: 'brainstorm' }).run(); - const reflectKindNodes = [ - ...(reflectEntryResult.nodes ?? []), - ...(brainstormEntryResult.nodes ?? []), - ]; - const reflectSessionResult = await read.query().match('reflect:*').run(); - const brainstormSessionResult = await read.query().match('brainstorm:*').run(); - const sessionNodes = [ - ...(reflectSessionResult.nodes ?? []), - ...(brainstormSessionResult.nodes ?? []), - ]; - const sessionNodeIds = new Set(sessionNodes.map((node) => node.id)); - const artifactResult = await read.query().match(`${ARTIFACT_PREFIX}*`).run(); - - const missingEdges = []; - const reflectNodes = new Map(); - addReflectNodes(reflectNodes, reflectKindNodes); - - for (const node of sessionNodes) { - // eslint-disable-next-line no-await-in-loop -- sequential migration fallback query per reflect session - const linkedReflectEntryResult = await read.query() - .match('entry:*') - .where({ sessionId: node.id }) - .run(); - addReflectNodes(reflectNodes, linkedReflectEntryResult.nodes); - } - - // Check capture edges — sequential per-node edge traversal - for (const node of captureResult.nodes ?? []) { - const { id, props } = node; - /* eslint-disable no-await-in-loop -- sequential migration edge checks */ - if (props.thoughtId) { - await pushMissingEdgeIfAbsent(read, missingEdges, id, props.thoughtId, 'expresses'); - } - if (props.sessionId) { - await pushMissingEdgeIfAbsent(read, missingEdges, id, props.sessionId, 'captured_in'); - } - /* eslint-enable no-await-in-loop */ - } - - // Check reflect session edges - const seedEntryIdBySessionId = new Map(); - for (const node of sessionNodes) { - const { id, props } = node; - if (props.seedEntryId) { - seedEntryIdBySessionId.set(id, props.seedEntryId); - // eslint-disable-next-line no-await-in-loop -- sequential migration - await pushMissingEdgeIfAbsent(read, missingEdges, id, props.seedEntryId, 'seeded_by'); - } - } - - // Check reflect entry edges - for (const node of reflectNodes.values()) { - const { id, props } = node; - const sessionId = props.sessionId ?? inferReflectSessionId(props, sessionNodes); - const seedEntryId = props.seedEntryId ?? seedEntryIdBySessionId.get(sessionId); - /* eslint-disable no-await-in-loop -- sequential migration edge checks */ - if (sessionId) { - await pushMissingEdgeIfAbsent(read, missingEdges, id, sessionId, 'produced_in', { - knownTargetNodeIds: sessionNodeIds, - }); - } - if (seedEntryId) { - await pushMissingEdgeIfAbsent(read, missingEdges, id, seedEntryId, 'responds_to'); - } - /* eslint-enable no-await-in-loop */ - } - - // Check artifact edges - for (const node of artifactResult.nodes ?? []) { - const { id, props } = node; - /* eslint-disable no-await-in-loop -- sequential migration edge checks */ - if (props.primaryInputKind === 'thought' && props.primaryInputId) { - await pushMissingEdgeIfAbsent(read, missingEdges, id, props.primaryInputId, 'derived_from'); - } - if (props.primaryInputKind === 'capture' && props.primaryInputId) { - await pushMissingEdgeIfAbsent(read, missingEdges, id, props.primaryInputId, 'contextualizes'); - } - /* eslint-enable no-await-in-loop */ - } - - // Build chronology chain - const captures = (captureResult.nodes ?? []) - .map((node) => ({ id: node.id, sortKey: String(node.props.sortKey || '') })) - .sort(compareEntriesNewestFirst); - - // Check older chain - for (let index = 0; index + 1 < captures.length; index += 1) { - // eslint-disable-next-line no-await-in-loop -- sequential chain check - await pushMissingEdgeIfAbsent(read, missingEdges, captures[index].id, captures[index + 1].id, 'older'); - } - - // Check classification nodes - const classificationNodesToCreate = []; - for (const name of CLASSIFICATIONS) { - const nodeId = `${CLASSIFICATION_PREFIX}${name}`; - // eslint-disable-next-line no-await-in-loop -- checking 7 standing nodes - const existing = await read.getNodeProps(nodeId); - if (!existing) { - classificationNodesToCreate.push({ nodeId, name }); - } - } + const memory = await openNativeMemory(repoDir); + const hadHistory = memory.hasHistory(); + const previous = hadHistory + ? decodeNativeDocument( + await memory.memoryDocument(GRAPH_META_ID), + 'memory-object' + ) + : null; + const previousVersion = previous?.graphModelVersion ?? null; + const timestamp = getCurrentTime().toISOString(); - if ( - missingEdges.length === 0 - && classificationNodesToCreate.length === 0 - && !needsMetadataNode - && !needsGraphVersionUpdate - ) { + if (previousVersion === GRAPH_MODEL_VERSION) { return Object.freeze({ changed: false, graphModelVersion: GRAPH_MODEL_VERSION, @@ -138,94 +34,45 @@ export async function migrateGraphModel(repoDir) { }); } - const timestamp = getCurrentTime().toISOString(); - const needsStandardPatch = classificationNodesToCreate.length > 0 - || needsMetadataNode - || needsGraphVersionUpdate; - - if (needsStandardPatch) { - await commitThinkWorldline(repoDir, (patch) => { - if (needsMetadataNode) { - patch - .addNode(GRAPH_META_ID) - .setProperty(GRAPH_META_ID, 'kind', 'graph_meta') - .setProperty(GRAPH_META_ID, 'createdAt', timestamp); - } - - patch - .setProperty(GRAPH_META_ID, 'graphModelVersion', GRAPH_MODEL_VERSION) - .setProperty(GRAPH_META_ID, 'updatedAt', timestamp); - - for (const { nodeId, name } of classificationNodesToCreate) { - patch - .addNode(nodeId) - .setProperty(nodeId, 'kind', 'classification') - .setProperty(nodeId, 'name', name) - .setProperty(nodeId, 'createdAt', timestamp); - } - }); - } + await memory.writeMemoryDocument({ + id: GRAPH_META_ID, + bytes: encodeNativeDocument('memory-object', { + ...previous, + id: GRAPH_META_ID, + kind: 'graph_meta', + graphModelVersion: GRAPH_MODEL_VERSION, + updatedAt: timestamp, + }), + declare: previous === null, + }); - if (missingEdges.length > 0) { - const migrationWriterId = `${worldline.writerId}.migration`; - await commitThinkWorldlineWithWriter(repoDir, migrationWriterId, (patch) => { - for (const edge of missingEdges) { - patch.addEdge(edge.from, edge.to, edge.label); - } + for (const name of CLASSIFICATIONS) { + const id = `${CLASSIFICATION_PREFIX}${name}`; + const exists = previous === null + ? false + // eslint-disable-next-line no-await-in-loop -- upgrades check the bounded object observer + : await memory.exists(id); + // eslint-disable-next-line no-await-in-loop -- standing ontology nodes are admitted one Intent at a time + await memory.writeMemoryDocument({ + id, + bytes: encodeNativeDocument('memory-object', { + id, + kind: 'classification', + name, + createdAt: timestamp, + }), + declare: !exists, }); } + await ensureEmptyNativeIndex(memory, 'capture', { + knownMissing: !hadHistory, + }); return Object.freeze({ changed: true, graphModelVersion: GRAPH_MODEL_VERSION, - edgesAdded: missingEdges.length, + edgesAdded: 0, edgesRemoved: 0, - metadataUpdated: needsMetadataNode || needsGraphVersionUpdate, - }); -} - -async function pushMissingEdgeIfAbsent(read, target, from, to, label, { knownTargetNodeIds = null } = {}) { - // Verify target node exists before checking edge - if (!knownTargetNodeIds?.has(to)) { - const targetProps = await read.getNodeProps(to); - if (!targetProps) { return; } - } - - const traversal = await read.query().match(from).outgoing(label).run(); - const hasEdge = (traversal.nodes ?? []).some((n) => n.id === to); - if (!hasEdge) { - target.push({ from, to, label }); - } -} - -function addReflectNodes(target, nodes = []) { - for (const node of nodes ?? []) { - if (!node?.id || !isReflectEntryProps(node.props ?? {})) { - continue; - } - target.set(node.id, node); - } -} - -function isReflectEntryProps(props) { - return props.kind === 'reflect' - || props.kind === 'brainstorm' - || props.source === 'reflect' - || props.source === 'brainstorm' - || typeof props.seedEntryId === 'string' - || typeof props.promptType === 'string'; -} - -function inferReflectSessionId(reflectProps, sessionNodes) { - const candidates = sessionNodes.filter(({ props }) => { - if (reflectProps.seedEntryId && props.seedEntryId !== reflectProps.seedEntryId) { - return false; - } - if (reflectProps.promptType && props.promptType && props.promptType !== reflectProps.promptType) { - return false; - } - return true; + metadataUpdated: true, }); - - return candidates.length === 1 ? candidates[0].id : null; } diff --git a/src/store/native-document.js b/src/store/native-document.js new file mode 100644 index 00000000..73137f38 --- /dev/null +++ b/src/store/native-document.js @@ -0,0 +1,44 @@ +import { ValidationError } from '../errors.js'; + +export const MEMORY_DOCUMENT_KEY = 'think.memory-document.v1'; +export const INDEX_DOCUMENT_KEY = 'think.memory-index.v1'; +export const INDEX_PAGE_DOCUMENT_KEY = 'think.memory-index-page.v1'; + +const DOCUMENT_VERSION = 1; +const decoder = new TextDecoder('utf-8', { fatal: true }); +const encoder = new TextEncoder(); + +export function encodeNativeDocument(kind, value) { + if (!isRecord(value)) { + throw new ValidationError(`Native ${kind} document must be an object`); + } + return encoder.encode(JSON.stringify({ + version: DOCUMENT_VERSION, + kind, + value, + })); +} + +export function decodeNativeDocument(bytes, expectedKind) { + if (!(bytes instanceof Uint8Array)) { + return null; + } + try { + const envelope = JSON.parse(decoder.decode(bytes)); + if ( + !isRecord(envelope) + || envelope.version !== DOCUMENT_VERSION + || envelope.kind !== expectedKind + || !isRecord(envelope.value) + ) { + return null; + } + return Object.freeze(envelope.value); + } catch { + return null; + } +} + +function isRecord(value) { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/src/store/native-index.js b/src/store/native-index.js new file mode 100644 index 00000000..e77ca331 --- /dev/null +++ b/src/store/native-index.js @@ -0,0 +1,539 @@ +import { mkdir, open, rm, stat } from 'node:fs/promises'; +import path from 'node:path'; + +import { ThinkError } from '../errors.js'; +import { READ_MODEL_PREFIX } from './constants.js'; +import { + decodeNativeDocument, + encodeNativeDocument, +} from './native-document.js'; +import { openNativeMemory } from './native-runtime.js'; + +export const NATIVE_INDEX_PAGE_SIZE = 64; +export const NATIVE_INDEX_READ_LIMIT = 4096; + +const LOCK_RETRY_MS = 20; +const LOCK_TIMEOUT_MS = 10_000; +const STALE_LOCK_MS = 60_000; +const INDEX_PREFIX = `${READ_MODEL_PREFIX}v19:index:`; + +export async function ensureEmptyNativeIndex(memory, kind, { + knownMissing = false, +} = {}) { + const id = indexId(kind); + if (!knownMissing && await memory.exists(id)) { + return false; + } + const state = emptyIndexState(kind); + await memory.writeMemoryIndex({ + id, + bytes: encodeNativeDocument('memory-index', state), + }); + return true; +} + +export async function appendIndexedMemoryObject(repoDir, { + id, + kind, + facts, +}) { + return await withWriteLock( + repoDir, + () => appendIndexedMemoryObjectLocked(repoDir, { id, kind, facts }) + ); +} + +export async function updateIndexedMemoryObject(repoDir, { + id, + kind, + facts, +}) { + return await withWriteLock( + repoDir, + () => updateIndexedMemoryObjectLocked(repoDir, { id, kind, facts }) + ); +} + +export async function listIndexedMemoryDocuments(repoDir, kind, { + limit = 50, + memory: providedMemory = null, +} = {}) { + const requested = normalizeLimit(limit); + if (requested === 0) { + return Object.freeze([]); + } + const memory = providedMemory ?? await openNativeMemory(repoDir); + if (!memory.hasHistory()) { + return Object.freeze([]); + } + const reader = await memory.captureBoundedReader(); + const state = await readIndexState(reader, kind); + if (!hasIndexEntries(state)) { + return Object.freeze([]); + } + + return await readRecentIndexPages(reader, state, requested); +} + +export async function readIndexedMemoryDocument(repoDir, id, { + kinds, + memory: providedMemory = null, +} = {}) { + const memory = providedMemory ?? await openNativeMemory(repoDir); + if (!memory.hasHistory()) { + return null; + } + const reader = await memory.captureBoundedReader(); + return await findDocumentAcrossKinds(reader, kinds, id); +} + +export async function readIndexedMemoryBatch(repoDir, kind, { + after = null, + limit = 500, +} = {}) { + const requested = normalizeLimit(limit); + if (requested === 0) { + return emptyBatch(after); + } + const memory = await openNativeMemory(repoDir); + if (!memory.hasHistory()) { + return emptyBatch(after); + } + const reader = await memory.captureBoundedReader(); + const state = await readIndexState(reader, kind); + if (!hasIndexEntries(state)) { + return emptyBatch(after); + } + + const priorCursor = await locateIndexCursor(reader, state, after); + return await readForwardIndexPages(reader, state, requested, priorCursor); +} + +export async function listIndexedMemoryIds(repoDir, kind, options = {}) { + const documents = await listIndexedMemoryDocuments(repoDir, kind, options); + return Object.freeze(documents.map(document => document.id)); +} + +export async function readNativeIndexSummary(repoDir, kind) { + const memory = await openNativeMemory(repoDir); + if (!memory.hasHistory()) { + return emptyIndexState(kind); + } + return await readIndexState(memory, kind); +} + +async function appendIndexedMemoryObjectLocked(repoDir, { id, kind, facts }) { + const memory = await openNativeMemory(repoDir); + const hadHistory = memory.hasHistory(); + const position = await loadAppendPosition(memory, kind, hadHistory); + const document = Object.freeze({ + id, + kind, + ...facts, + previousKindId: position.previousId, + }); + const nextPage = Object.freeze({ + ...position.page, + entries: Object.freeze([...position.page.entries, document]), + }); + const nextState = Object.freeze({ + ...position.state, + total: position.state.total + 1, + latestId: id, + headPage: position.pageNumber, + }); + await persistIndexAppend(memory, position, nextPage, nextState); + if (!hadHistory) { + await memory.repairBasis(); + } + return Object.freeze({ + document, + indexId: position.state.id, + pageId: position.pageId, + previousId: position.previousId, + total: nextState.total, + }); +} + +async function loadAppendPosition(memory, kind, hadHistory) { + const state = hadHistory + ? await readIndexState(memory, kind) + : emptyIndexState(kind); + const pageNumber = Math.floor(state.total / NATIVE_INDEX_PAGE_SIZE); + const page = state.total % NATIVE_INDEX_PAGE_SIZE === 0 + ? emptyPage(kind, pageNumber) + : await readIndexPage(memory, kind, pageNumber); + return Object.freeze({ + page, + pageId: indexPageId(kind, pageNumber), + pageNumber, + previousId: page.entries.at(-1)?.id + ?? await previousPageLastId(memory, kind, pageNumber), + state, + }); +} + +async function persistIndexAppend(memory, position, page, state) { + await memory.writeMemoryIndexPage({ + id: position.pageId, + bytes: encodeNativeDocument('memory-index-page', page), + declare: position.page.entries.length === 0, + }); + await memory.writeMemoryIndex({ + id: position.state.id, + bytes: encodeNativeDocument('memory-index', state), + declare: !position.state.exists, + }); +} + +async function updateIndexedMemoryObjectLocked(repoDir, { id, kind, facts }) { + const memory = await openNativeMemory(repoDir); + const located = await locateIndexedObjectForUpdate(memory, { id, kind }); + const document = Object.freeze({ + ...located.current, + ...facts, + id, + kind, + }); + const nextPage = Object.freeze({ + ...located.page, + entries: Object.freeze(located.page.entries.map(entry => ( + entry.id === id ? document : entry + ))), + }); + const pageId = indexPageId(kind, located.page.pageNumber); + await memory.writeMemoryIndexPage({ + id: pageId, + bytes: encodeNativeDocument('memory-index-page', nextPage), + declare: false, + }); + return Object.freeze({ + document, + indexId: located.state.id, + pageId, + }); +} + +async function locateIndexedObjectForUpdate(memory, { id, kind }) { + requireIndexHistory(memory, id, kind); + const state = await readIndexState(memory, kind); + const page = await findIndexPage(memory, state, id); + if (!page) { + throw new ThinkError( + `Native ${kind} index does not contain ${id}`, + 'V19_INDEXED_OBJECT_NOT_INDEXED' + ); + } + const current = page.entries.find(entry => entry.id === id); + if (!current || current.kind !== kind) { + throw missingIndexedObjectError(id, kind); + } + return Object.freeze({ current, page, state }); +} + +function requireIndexHistory(memory, id, kind) { + if (!memory.hasHistory()) { + throw missingIndexedObjectError(id, kind); + } +} + +function missingIndexedObjectError(id, kind) { + return new ThinkError( + `Cannot update missing native ${kind} object ${id}`, + 'V19_INDEXED_OBJECT_MISSING' + ); +} + +async function readRecentIndexPages(reader, state, requested) { + const documents = []; + const pageNumbers = recentIndexPageNumbers(state, requested); + const pages = await Promise.all( + pageNumbers.map(pageNumber => readIndexPage(reader, state.kind, pageNumber)) + ); + for (const page of pages) { + const remaining = requested - documents.length; + documents.push(...page.entries.slice(-remaining).reverse()); + } + return Object.freeze(documents); +} + +function recentIndexPageNumbers(state, requested) { + const oldestRequestedOffset = Math.max(0, state.total - requested); + const oldestRequestedPage = Math.floor( + oldestRequestedOffset / NATIVE_INDEX_PAGE_SIZE + ); + return Object.freeze( + Array.from( + { length: state.headPage - oldestRequestedPage + 1 }, + (_, offset) => state.headPage - offset + ) + ); +} + +async function findDocumentAcrossKinds(reader, kinds, id) { + /* eslint-disable no-await-in-loop -- one coordinate checks a bounded kind and page set */ + for (const kind of kinds) { + const state = await readIndexState(reader, kind); + const page = state.exists && state.total > 0 + ? await findIndexPage(reader, state, id) + : null; + const document = page?.entries.find(entry => entry.id === id) ?? null; + if (document !== null) { + return document; + } + } + /* eslint-enable no-await-in-loop */ + return null; +} + +async function readForwardIndexPages(reader, state, requested, priorCursor) { + const documents = []; + let pageNumber = priorCursor?.pageNumber ?? 0; + let offset = (priorCursor?.offset ?? -1) + 1; + let cursor = priorCursor; + /* eslint-disable no-await-in-loop -- one bounded batch reads persisted index pages */ + while (pageNumber <= state.headPage && documents.length < requested) { + const page = await readIndexPage(reader, state.kind, pageNumber); + const remaining = requested - documents.length; + const selected = page.entries.slice(offset, offset + remaining); + documents.push(...selected); + cursor = selected.length > 0 + ? indexCursor(selected.at(-1), pageNumber, offset + selected.length - 1) + : cursor; + pageNumber += 1; + offset = 0; + } + /* eslint-enable no-await-in-loop */ + return Object.freeze({ + documents: Object.freeze(documents), + cursor, + }); +} + +function indexId(kind) { + return `${INDEX_PREFIX}${kind}`; +} + +function indexPageId(kind, number) { + return `${INDEX_PREFIX}${kind}:page:${String(number).padStart(8, '0')}`; +} + +async function readIndexState(memory, kind) { + const id = indexId(kind); + const decoded = decodeNativeDocument( + await memory.memoryIndex(id), + 'memory-index' + ); + if (!decoded) { + return emptyIndexState(kind); + } + return Object.freeze({ + id, + kind, + exists: true, + headPage: integerOrZero(decoded.headPage), + latestId: stringOrNull(decoded.latestId), + total: integerOrZero(decoded.total), + }); +} + +async function readIndexPage(memory, kind, pageNumber) { + const decoded = decodeNativeDocument( + await memory.memoryIndexPage(indexPageId(kind, pageNumber)), + 'memory-index-page' + ); + if (!decoded || !Array.isArray(decoded.entries)) { + return emptyPage(kind, pageNumber); + } + return Object.freeze({ + kind, + pageNumber, + entries: Object.freeze(decoded.entries.filter(isMemoryDocument)), + }); +} + +async function previousPageLastId(memory, kind, pageNumber) { + if (pageNumber === 0) { + return null; + } + const previous = await readIndexPage(memory, kind, pageNumber - 1); + return previous.entries.at(-1)?.id ?? null; +} + +async function findIndexPage(memory, state, id) { + /* eslint-disable no-await-in-loop -- indexed replacement scans bounded persisted pages */ + for (let pageNumber = state.headPage; pageNumber >= 0; pageNumber -= 1) { + const page = await readIndexPage(memory, state.kind, pageNumber); + if (page.entries.some(entry => entry.id === id)) { + return page; + } + } + /* eslint-enable no-await-in-loop */ + return null; +} + +async function locateIndexCursor(memory, state, after) { + const id = stringOrNull(after?.id); + if (id === null) { + return null; + } + if (isReusableIndexCursor(after, state)) { + const cursorPage = await readIndexPage(memory, state.kind, after.pageNumber); + const cursor = cursorForPage(cursorPage, id, after.offset); + if (cursor) { + return cursor; + } + } + + const page = await findIndexPage(memory, state, id); + if (page) { + return cursorForPage(page, id); + } + throw new ThinkError( + `Native ${state.kind} index does not contain cursor ${id}`, + 'V19_INDEX_CURSOR_MISSING' + ); +} + +function isReusableIndexCursor(cursor, state) { + if (!Number.isInteger(cursor?.pageNumber)) { + return false; + } + if (cursor.pageNumber < 0 || cursor.pageNumber > state.headPage) { + return false; + } + return Number.isInteger(cursor.offset) && cursor.offset >= 0; +} + +function cursorForPage(page, id, knownOffset = null) { + const offset = knownOffset ?? page.entries.findIndex(entry => entry.id === id); + if (page.entries[offset]?.id !== id) { + return null; + } + return indexCursor(page.entries[offset], page.pageNumber, offset); +} + +function indexCursor(document, pageNumber, offset) { + return Object.freeze({ + id: document.id, + pageNumber, + offset, + }); +} + +function emptyBatch(after) { + const id = stringOrNull(after?.id); + const cursor = id === null + ? null + : Object.freeze({ + id, + pageNumber: integerOrZero(after?.pageNumber), + offset: integerOrZero(after?.offset), + }); + return Object.freeze({ + documents: Object.freeze([]), + cursor, + }); +} + +function emptyIndexState(kind) { + return Object.freeze({ + id: indexId(kind), + kind, + exists: false, + headPage: 0, + latestId: null, + total: 0, + }); +} + +function emptyPage(kind, pageNumber) { + return Object.freeze({ + kind, + pageNumber, + entries: Object.freeze([]), + }); +} + +async function withWriteLock(repoDir, task) { + await mkdir(repoDir, { recursive: true }); + const lockPath = path.join(repoDir, '.think-v19-write.lock'); + const startedAt = Date.now(); + let handle = null; + + /* eslint-disable no-await-in-loop -- atomic lock acquisition requires bounded polling */ + while (handle === null) { + try { + handle = await open(lockPath, 'wx'); + await handle.writeFile(`${process.pid}\n${new Date().toISOString()}\n`); + } catch (error) { + if (error?.code !== 'EEXIST') { + throw error; + } + if (await isStaleLock(lockPath)) { + await rm(lockPath, { force: true }); + continue; + } + if (Date.now() - startedAt >= LOCK_TIMEOUT_MS) { + throw new ThinkError( + `Timed out waiting for the native Think writer lock at ${lockPath}`, + 'V19_WRITE_LOCK_TIMEOUT' + ); + } + await delay(LOCK_RETRY_MS); + } + } + /* eslint-enable no-await-in-loop */ + + try { + return await task(); + } finally { + await handle.close(); + await rm(lockPath, { force: true }); + } +} + +async function isStaleLock(lockPath) { + try { + const details = await stat(lockPath); + return Date.now() - details.mtimeMs >= STALE_LOCK_MS; + } catch (error) { + if (error?.code === 'ENOENT') { + return false; + } + throw error; + } +} + +function normalizeLimit(limit) { + if (!Number.isInteger(limit) || limit < 0) { + return 50; + } + return Math.min(limit, NATIVE_INDEX_READ_LIMIT); +} + +function hasIndexEntries(state) { + return state.exists && state.total > 0; +} + +function stringOrNull(value) { + return typeof value === 'string' && value.length > 0 ? value : null; +} + +function integerOrZero(value) { + return Number.isInteger(value) && value >= 0 ? value : 0; +} + +function isMemoryDocument(value) { + return typeof value === 'object' + && value !== null + && !Array.isArray(value) + && typeof value.id === 'string' + && typeof value.kind === 'string'; +} + +function delay(milliseconds) { + return new Promise(resolve => { + setTimeout(resolve, milliseconds); + }); +} diff --git a/src/store/native-runtime.js b/src/store/native-runtime.js new file mode 100644 index 00000000..74fb6f32 --- /dev/null +++ b/src/store/native-runtime.js @@ -0,0 +1,363 @@ +import { execFile, spawnSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { promisify } from 'node:util'; + +import { Runtime } from '@git-stunts/git-warp'; +import { captureCoordinate as captureLaneCoordinate } from '@git-stunts/git-warp/advanced'; + +import { ThinkError } from '../errors.js'; +import { GIT_BINARY, THINK_GIT_CONFIG_ARGS } from '../git.js'; +import { thinkMemory } from '../generated/think-memory.generated.js'; +import { GRAPH_NAME } from './constants.js'; +import { createWriterId } from './model.js'; +import { + INDEX_DOCUMENT_KEY, + INDEX_PAGE_DOCUMENT_KEY, +} from './native-document.js'; + +const execFileAsync = promisify(execFile); +const nodeRequire = createRequire(import.meta.url); +const packageRoot = path.dirname(nodeRequire.resolve('@git-stunts/git-warp/package.json')); +const gitWarpCli = path.join(packageRoot, 'bin', 'git-warp'); +const sessions = new Map(); +const MAX_WRITE_ATTEMPTS = 3; + +export async function openNativeMemory(repoDir, { + writerId = createWriterId(), +} = {}) { + const cacheKey = `${path.resolve(repoDir)}\0${writerId}`; + let session = sessions.get(cacheKey); + if (!session) { + // eslint-disable-next-line no-use-before-define -- the cache factory precedes the session class + session = await NativeMemorySession.open(repoDir, writerId); + sessions.set(cacheKey, session); + } + return session; +} + +export async function closeNativeMemory(repoDir) { + const resolvedRepo = path.resolve(repoDir); + const matches = [...sessions.entries()] + .filter(([cacheKey]) => cacheKey.startsWith(`${resolvedRepo}\0`)); + await Promise.all(matches.map(async ([cacheKey, session]) => { + sessions.delete(cacheKey); + await session.close(); + })); +} + +export class NativeMemoryCoordinateReader { + constructor(optic) { + this.optic = optic; + } + + async memoryIndex(subject) { + return await this.readProperty( + subject, + INDEX_DOCUMENT_KEY, + 'memory index' + ); + } + + async memoryIndexPage(subject) { + return await this.readProperty( + subject, + INDEX_PAGE_DOCUMENT_KEY, + 'memory index page' + ); + } + + async readProperty(subject, key, kind) { + const result = await this.optic.node(subject).prop(key).read(); + return normalizeObservedBytes( + result.exists ? result.value : null, + subject, + kind + ); + } +} + +export class NativeMemorySession { + constructor(repoDir, writerId, runtime, lane) { + this.repoDir = path.resolve(repoDir); + this.writerId = writerId; + this.runtime = runtime; + this.lane = lane; + } + + static async open(repoDir, writerId) { + const runtime = await Runtime.open({ + at: repoDir, + writer: writerId, + }); + const lane = await runtime.lane(GRAPH_NAME); + return new NativeMemorySession(repoDir, writerId, runtime, lane); + } + + async close() { + await this.runtime.close(); + } + + async reopen() { + await this.runtime.close(); + this.runtime = await Runtime.open({ + at: this.repoDir, + writer: this.writerId, + }); + this.lane = await this.runtime.lane(GRAPH_NAME); + } + + hasHistory() { + const result = spawnSync( + GIT_BINARY, + [ + ...THINK_GIT_CONFIG_ARGS, + '-C', + this.repoDir, + 'for-each-ref', + '--count=1', + '--format=%(refname)', + `refs/warp/${GRAPH_NAME}/writers/`, + ], + { encoding: 'utf8' } + ); + if (result.status !== 0) { + throw new ThinkError( + `Unable to inspect native Think writer refs in ${this.repoDir}`, + 'V19_HISTORY_INSPECTION_FAILED' + ); + } + return result.stdout.trim().length > 0; + } + + async declareMemoryObject(subject) { + return await this.write(thinkMemory.intents.declareMemoryObject({ subject })); + } + + async storeMemoryDocument(subject, value) { + return await this.write(thinkMemory.intents.storeMemoryDocument({ subject, value })); + } + + async storeMemoryIndex(subject, value) { + return await this.write(thinkMemory.intents.storeMemoryIndex({ subject, value })); + } + + async storeMemoryIndexPage(subject, value) { + return await this.write(thinkMemory.intents.storeMemoryIndexPage({ subject, value })); + } + + async memoryDocument(subject) { + return normalizeObservedBytes( + await this.observeOne(thinkMemory.observers.memoryDocument({ subject })), + subject, + 'memory document' + ); + } + + async memoryIndex(subject) { + return normalizeObservedBytes( + await this.observeOne(thinkMemory.observers.memoryIndex({ subject })), + subject, + 'memory index' + ); + } + + async memoryIndexPage(subject) { + return normalizeObservedBytes( + await this.observeOne(thinkMemory.observers.memoryIndexPage({ subject })), + subject, + 'memory index page' + ); + } + + async writeMemoryDocument({ + id, + bytes, + declare = true, + }) { + return await this.writeDocument({ + bytes, + declare, + id, + store: value => this.storeMemoryDocument(id, value), + }); + } + + async writeMemoryIndex({ + id, + bytes, + declare = true, + }) { + return await this.writeDocument({ + bytes, + declare, + id, + store: value => this.storeMemoryIndex(id, value), + }); + } + + async writeMemoryIndexPage({ + id, + bytes, + declare = true, + }) { + return await this.writeDocument({ + bytes, + declare, + id, + store: value => this.storeMemoryIndexPage(id, value), + }); + } + + async writeDocument({ id, bytes, declare, store }) { + const receipts = []; + if (declare) { + receipts.push(await this.declareMemoryObject(id)); + } + receipts.push(await store(bytes)); + return Object.freeze(receipts); + } + + async exists(subject) { + return await this.observeOne(thinkMemory.observers.memoryObjectExists({ subject })); + } + + async observeOne(observer) { + const values = await this.consumeObservation(observer); + if (values.length !== 1) { + throw new ThinkError( + `Observer ${observer.id} emitted ${values.length} readings; expected exactly one`, + 'V19_OBSERVER_CARDINALITY' + ); + } + return values[0]; + } + + async consumeObservation(observer) { + try { + return await this.consumeObservationOnce(observer); + } catch (error) { + if (!isMissingBasisError(error)) { + throw error; + } + await this.repairBasis(); + return await this.consumeObservationOnce(observer); + } + } + + async captureBoundedReader() { + let coordinate; + try { + coordinate = await captureLaneCoordinate(this.lane); + } catch (error) { + if (!isMissingCoordinateBasisError(error)) { + throw error; + } + await this.repairBasis(); + coordinate = await captureLaneCoordinate(this.lane); + } + return new NativeMemoryCoordinateReader(coordinate.optic()); + } + + async consumeObservationOnce(observer) { + const observation = this.lane.observe(observer); + const values = []; + for await (const reading of observation) { + values.push(reading.value); + } + const receipt = await observation.receipt; + if (receipt.status !== 'completed') { + const error = new ThinkError( + `Observer ${observer.id} ${receipt.status}: ${receipt.reason ?? 'no reason provided'}`, + 'V19_OBSERVATION_INCOMPLETE' + ); + error.receipt = receipt; + throw error; + } + return Object.freeze(values); + } + + async repairBasis() { + await this.runtime.close(); + await execFileAsync(process.execPath, [ + gitWarpCli, + '--repo', + this.repoDir, + '--lane', + GRAPH_NAME, + '--writer', + this.writerId, + '--json', + 'repair', + '--action', + 'materialization', + ]); + this.runtime = await Runtime.open({ + at: this.repoDir, + writer: this.writerId, + }); + this.lane = await this.runtime.lane(GRAPH_NAME); + } + + async write(intent) { + let attempt = 1; + /* eslint-disable no-await-in-loop -- admission retries must follow the current writer head */ + while (true) { + try { + const receipt = await this.lane.write(intent); + requireAdmitted(receipt); + return receipt; + } catch (error) { + if (!isWriterCasConflict(error) || attempt >= MAX_WRITE_ATTEMPTS) { + throw error; + } + await this.reopen(); + attempt += 1; + } + } + /* eslint-enable no-await-in-loop */ + } +} + +function requireAdmitted(receipt) { + if (receipt.outcome.kind === 'derived' || receipt.outcome.kind === 'plural') { + return; + } + const error = new ThinkError( + `Native Think write was ${receipt.outcome.kind}`, + 'V19_WRITE_NOT_ADMITTED' + ); + error.receipt = receipt; + throw error; +} + +function isWriterCasConflict(error) { + return error instanceof Error && ( + error.code === 'E_WRITER_CAS_CONFLICT' + || error.message.includes('writer ref was updated by another process') + ); +} + +function isMissingBasisError(error) { + return error instanceof Error + && error.code === 'V19_OBSERVATION_INCOMPLETE' + && error.receipt?.reason === 'missing_bounded_basis'; +} + +function isMissingCoordinateBasisError(error) { + return error instanceof Error + && error.code === 'E_OPTIC_NO_BOUNDED_BASIS'; +} + +function normalizeObservedBytes(value, subject, kind) { + if (value === null || value instanceof Uint8Array) { + return value; + } + if (typeof value?.toUint8Array === 'function') { + return value.toUint8Array(); + } + throw new ThinkError( + `Native Think ${kind} for ${subject} returned an unsupported value`, + 'V19_BYTES_VALUE_INVALID' + ); +} diff --git a/src/store/queries.js b/src/store/queries.js index 66df981a..e6a339f3 100644 --- a/src/store/queries.js +++ b/src/store/queries.js @@ -1,5 +1,4 @@ import { getPromptMetricsFile } from '../paths.js'; -import { KEYWORD_PREFIX } from './constants.js'; import { compareEntriesNewestFirst, formatBucketKey, @@ -26,29 +25,26 @@ import { getLatestCaptureId, getStoredEntry, getChronologyNeighborEntries, + getIndexSummary, listIndexedCaptureProps, listChronologyEntries, + listEntriesByKind, listRecentStoredEntries, openProductReadHandle, resolveHistorySessionTraversal, toBrowseEntry, } from './runtime.js'; -import { - CAPTURE_READ_MODEL_LIMIT, - readCaptureReadModel, -} from './read-model.js'; import { assessReflectability, - ensureFirstDerivedArtifacts, getCanonicalThought, getSeedQualityReceipt, getSessionAttributionReceipt, - getSessionAttributionReceiptIfPresent, listDirectDerivedReceipts, } from './derivation.js'; import { KeywordTrie } from './trie.js'; const DEFAULT_RECENT_LIMIT = 50; +const CAPTURE_READ_MODEL_LIMIT = 500; const searchIndexCache = new Map(); const searchIndexLoadingPromises = new Map(); @@ -76,10 +72,10 @@ export function loadSearchIndex(repoDir) { const read = await openProductReadHandle(repoDir); const trie = new KeywordTrie(); - const keywordResult = await read.view.query().match(`${KEYWORD_PREFIX}*`).where({ kind: 'keyword' }).run(); - for (const node of keywordResult.nodes ?? []) { - if (node.props.name) { - trie.insert(node.props.name); + const keywords = await listEntriesByKind(read, 'keyword'); + for (const keyword of keywords) { + if (keyword.name) { + trie.insert(keyword.name); } } @@ -259,14 +255,14 @@ export async function getPromptMetrics({ from, to, since, bucket } = {}) { export async function listRecent(repoDir, { count = null, query = null } = {}) { const limit = count ?? DEFAULT_RECENT_LIMIT; const read = await openProductReadHandle(repoDir); - const index = await readCaptureReadModel(read); + const index = await getIndexSummary(read, 'capture'); if (!query) { const unfilteredRecent = (await listRecentStoredEntries(read, { limit })) .map(toBrowseEntry); return Object.freeze({ entries: unfilteredRecent, - total: index.totalCaptures, + total: index.total, }); } @@ -359,15 +355,12 @@ export async function getBrowseWindowForRead(read, entryId) { } export async function inspectRawEntryForRead(read, entryId) { - let entry = await getStoredEntry(read, entryId); + const entry = await getStoredEntry(read, entryId); if (!entry || entry.kind !== 'capture') { return null; } - await ensureFirstDerivedArtifacts(read.repoDir, read, entry); - entry = await getStoredEntry(read, entryId); - const canonicalThought = await getCanonicalThought(read, entry); const seedQuality = await getSeedQualityReceipt(read, entry); const sessionAttribution = await getSessionAttributionReceipt(read, entry); @@ -392,22 +385,15 @@ export async function inspectRawEntryForRead(read, entryId) { } async function listAnnotationsForEntry(read, entryId) { - const traversal = await read.view.query().match(entryId).incoming('annotates').run(); - const annotations = []; - - for (const node of traversal.nodes ?? []) { - // eslint-disable-next-line no-await-in-loop -- sequential annotation reads - const entry = await getStoredEntry(read, node.id); - if (entry) { - annotations.push(Object.freeze({ - annotationId: entry.id, - text: entry.text, - createdAt: entry.createdAt, - })); - } - } - - return annotations.sort((a, b) => a.createdAt.localeCompare(b.createdAt)); + const entries = await listEntriesByKind(read, 'annotation'); + return entries + .filter(entry => entry.targetEntryId === entryId) + .map(entry => Object.freeze({ + annotationId: entry.id, + text: entry.text, + createdAt: entry.createdAt, + })) + .sort((left, right) => left.createdAt.localeCompare(right.createdAt)); } async function buildBrowseWindow(read, entryId) { @@ -417,7 +403,7 @@ async function buildBrowseWindow(read, entryId) { return null; } - const sessionAttribution = await getSessionAttributionReceiptIfPresent(read, currentEntry); + const sessionAttribution = await getSessionAttributionReceipt(read, currentEntry); const current = toBrowseEntryWithSession(currentEntry, sessionAttribution?.sessionId ?? null); const neighbors = await resolveChronologyNeighbors(read, currentEntry); const older = neighbors.older ? toBrowseEntry(neighbors.older) : null; diff --git a/src/store/read-model.js b/src/store/read-model.js deleted file mode 100644 index 59c70536..00000000 --- a/src/store/read-model.js +++ /dev/null @@ -1,490 +0,0 @@ -import { createHash } from 'node:crypto'; - -import { parseJson, stringifyJson } from '../json.js'; -import { - CAPTURE_READ_MODEL_ID, - EXACT_READ_UNAVAILABLE, - READ_MODEL_PREFIX, - SCHEMA_VERSION, -} from './constants.js'; -import { compareEntriesNewestFirst } from './model.js'; - -export const CAPTURE_READ_MODEL_LIMIT = 500; -export const AMBIENT_READ_MODEL_LIMIT = 250; -export const AMBIENT_INDEX_KEYS = Object.freeze([ - 'ambientGitRemote', - 'ambientGitRoot', - 'ambientCwd', -]); -const CAPTURE_RECORD_STRING_FIELDS = Object.freeze([ - 'writerId', - 'source', - 'channel', - 'thoughtId', - 'sessionId', -]); -const CAPTURE_RECORD_AMBIENT_FIELDS = Object.freeze([ - ['ambientCwd', 'cwd'], - ['ambientGitRoot', 'gitRoot'], - ['ambientGitRemote', 'gitRemote'], - ['ambientGitBranch', 'gitBranch'], -]); -const CAPTURE_RECORD_PROVENANCE_FIELDS = Object.freeze([ - ['captureIngress', 'ingress'], - ['captureSourceApp', 'sourceApp'], - ['captureSourceURL', 'sourceURL'], -]); - -export function ambientReadModelId(key, value) { - const fingerprint = createHash('sha256') - .update(`${key}\0${value}`, 'utf8') - .digest('hex'); - return `${READ_MODEL_PREFIX}ambient:${key}:${fingerprint}`; -} - -export async function readCaptureReadModel(read, { preferFast = true, useExact = true } = {}) { - const props = await readCaptureReadModelProps(read, { preferFast, useExact }); - return normalizeCaptureReadModel(props); -} - -export async function readCaptureRecords(read, { limit = CAPTURE_READ_MODEL_LIMIT } = {}) { - const index = await readCaptureReadModel(read); - return index.records.slice(0, normalizeLimit(limit, CAPTURE_READ_MODEL_LIMIT)); -} - -export async function readAmbientReadModel(read, key, value) { - if (!key || !value) { - return emptyAmbientReadModel(key, value); - } - - const props = await read.view.getNodeProps(ambientReadModelId(key, value)); - return normalizeAmbientReadModel(key, value, props); -} - -export async function readAmbientCaptureRefs(read, scope, { limit = AMBIENT_READ_MODEL_LIMIT } = {}) { - const refs = new Map(); - - for (const { key, value } of ambientScopeDescriptors(scope)) { - // eslint-disable-next-line no-await-in-loop -- exact bounded read per ambient index key - const index = await readAmbientReadModel(read, key, value); - for (const ref of index.refs) { - refs.set(ref.id, ref); - } - } - - return [...refs.values()] - .sort(compareEntriesNewestFirst) - .slice(0, normalizeLimit(limit, AMBIENT_READ_MODEL_LIMIT)); -} - -export async function applyCaptureReadModelPatch(patch, read, entry, { - ambientContext = null, -} = {}) { - const ref = normalizeCaptureRef(entry); - const record = normalizeCaptureRecord(entry, { ambientContext }); - if (!ref) { - return Object.freeze({ previousLatestRef: null }); - } - - const captureIndex = await readCaptureReadModel(read, { preferFast: false, useExact: false }); - const nextCaptureIndex = addCaptureToCaptureReadModel(captureIndex, ref, record); - writeCaptureReadModelPatch(patch, nextCaptureIndex); - - return Object.freeze({ - previousLatestRef: captureIndex.refs.find((candidate) => candidate.id !== ref.id) ?? null, - }); -} - -export function applyPendingCaptureReadModelPatch(patch, entry, { - ambientContext = null, -} = {}) { - const ref = normalizeCaptureRef(entry); - const record = normalizeCaptureRecord(entry, { ambientContext }); - if (!ref) { - return; - } - - patch - .addNode(CAPTURE_READ_MODEL_ID) - .setProperty(CAPTURE_READ_MODEL_ID, 'kind', 'capture_read_model') - .setProperty(CAPTURE_READ_MODEL_ID, 'schemaVersion', SCHEMA_VERSION) - .setProperty(CAPTURE_READ_MODEL_ID, 'latestCaptureId', ref.id) - .setProperty(CAPTURE_READ_MODEL_ID, 'latestCaptureRefJson', stringifyJson(ref)); - - if (record) { - patch - .setProperty(CAPTURE_READ_MODEL_ID, 'latestCaptureRecordJson', stringifyJson(record)) - .setProperty(CAPTURE_READ_MODEL_ID, 'fastCaptureRecordsJson', stringifyJson([record])); - } -} - -export function normalizeCaptureRef(entry) { - if (!entry?.id || entry.kind !== 'capture') { - return null; - } - - return Object.freeze({ - id: entry.id, - createdAt: entry.createdAt ?? null, - sortKey: String(entry.sortKey ?? ''), - }); -} - -export function normalizeCaptureRecord(entry, { ambientContext = null } = {}) { - const ref = normalizeCaptureRef(entry); - if (!ref) { - return null; - } - - const record = { - ...ref, - kind: 'capture', - text: textOrEmpty(entry.text), - }; - - assignStringFields(record, entry, CAPTURE_RECORD_STRING_FIELDS); - assignAmbientFields(record, entry, ambientContext); - assignProvenanceFields(record, entry); - - return Object.freeze(record); -} - -export function captureRecordToProps(record) { - const props = { - kind: 'capture', - createdAt: record.createdAt, - sortKey: record.sortKey, - }; - - assignStringFields(props, record, CAPTURE_RECORD_STRING_FIELDS); - assignStringFields(props, record, CAPTURE_RECORD_AMBIENT_FIELDS.map(([field]) => field)); - assignStringFields(props, record, CAPTURE_RECORD_PROVENANCE_FIELDS.map(([field]) => field)); - - return Object.freeze(props); -} - -async function readCaptureReadModelProps(read, { preferFast, useExact }) { - if (useExact && typeof read.readNodeProp === 'function') { - const recordsJson = await read.readNodeProp( - CAPTURE_READ_MODEL_ID, - preferFast ? 'fastCaptureRecordsJson' : 'recentCaptureRecordsJson' - ); - if (recordsJson === EXACT_READ_UNAVAILABLE) { - return normalizeReadModelPropsPreference( - await read.view.getNodeProps(CAPTURE_READ_MODEL_ID), - { preferFast } - ); - } - if (recordsJson !== undefined) { - return { - kind: 'capture_read_model', - recentCaptureRecordsJson: recordsJson, - }; - } - - return null; - } - - return await read.view.getNodeProps(CAPTURE_READ_MODEL_ID); -} - -function normalizeReadModelPropsPreference(props, { preferFast }) { - if (!props || !preferFast || props.fastCaptureRecordsJson === undefined) { - return props; - } - - return { - ...props, - recentCaptureRecordsJson: props.fastCaptureRecordsJson, - }; -} - -function normalizeCaptureReadModel(props) { - const persistedRecords = parseRecords(props?.recentCaptureRecordsJson); - const pendingLatestRecord = parseRecord(props?.latestCaptureRecordJson); - const records = mergeRecords(persistedRecords, pendingLatestRecord, CAPTURE_READ_MODEL_LIMIT); - const persistedRefs = parseRefs(props?.recentCaptureRefsJson); - const pendingLatestRef = parseRef(props?.latestCaptureRefJson); - const refs = records.length > 0 - ? records.map(captureRecordToRef) - : mergePersistedAndPendingRefs(persistedRefs, pendingLatestRef); - - return Object.freeze({ - id: CAPTURE_READ_MODEL_ID, - kind: 'capture_read_model', - persistedRefs: Object.freeze(persistedRefs), - persistedRecords: Object.freeze(persistedRecords), - refs: Object.freeze(refs), - records: Object.freeze(records), - latestCaptureId: latestCaptureIdFromProps(props, refs), - totalCaptures: totalCapturesFromProps(props, { persistedRefs, persistedRecords }), - }); -} - -function mergePersistedAndPendingRefs(persistedRefs, pendingLatestRef) { - if (!pendingLatestRef) { - return mergeRefs(persistedRefs, null, CAPTURE_READ_MODEL_LIMIT); - } - - return mergeRefs([...persistedRefs, pendingLatestRef], null, CAPTURE_READ_MODEL_LIMIT); -} - -function latestCaptureIdFromProps(props, refs) { - return props?.latestCaptureId ?? refs[0]?.id ?? null; -} - -function totalCapturesFromProps(props, { persistedRefs, persistedRecords }) { - if (Number.isInteger(props?.totalCaptures)) { - return props.totalCaptures; - } - - return Math.max(persistedRefs.length, persistedRecords.length); -} - -function normalizeAmbientReadModel(key, value, props) { - const records = parseRecords(props?.recentCaptureRecordsJson).slice(0, AMBIENT_READ_MODEL_LIMIT); - const refs = parseRefs(props?.recentCaptureRefsJson).slice(0, AMBIENT_READ_MODEL_LIMIT); - return Object.freeze({ - id: ambientReadModelId(key, value), - kind: 'ambient_capture_read_model', - ambientKey: key, - ambientValue: value, - refs: Object.freeze(refs), - records: Object.freeze(records), - }); -} - -function emptyAmbientReadModel(key, value) { - return Object.freeze({ - id: value ? ambientReadModelId(key, value) : null, - kind: 'ambient_capture_read_model', - ambientKey: key, - ambientValue: value, - refs: Object.freeze([]), - records: Object.freeze([]), - }); -} - -function addCaptureToCaptureReadModel(index, ref, record) { - const refs = mergeRefs(index.refs, ref, CAPTURE_READ_MODEL_LIMIT); - const records = record - ? mergeRecords(index.records, record, CAPTURE_READ_MODEL_LIMIT) - : index.records; - const alreadyIndexed = index.persistedRefs.some((candidate) => candidate.id === ref.id) || - index.persistedRecords.some((candidate) => candidate.id === ref.id); - return Object.freeze({ - ...index, - refs: Object.freeze(refs), - records: Object.freeze(records), - latestCaptureId: refs[0]?.id ?? null, - totalCaptures: alreadyIndexed ? index.totalCaptures : index.totalCaptures + 1, - }); -} - -function mergeRefs(refs, ref, limit) { - const byId = new Map(refs.map((candidate) => [candidate.id, candidate])); - if (ref) { - byId.set(ref.id, ref); - } - return [...byId.values()] - .sort(compareEntriesNewestFirst) - .slice(0, limit); -} - -function writeCaptureReadModelPatch(patch, index) { - patch - .addNode(index.id) - .setProperty(index.id, 'kind', index.kind) - .setProperty(index.id, 'schemaVersion', SCHEMA_VERSION) - .setProperty(index.id, 'latestCaptureId', index.latestCaptureId) - .setProperty(index.id, 'totalCaptures', index.totalCaptures) - .setProperty(index.id, 'recentCaptureRefsJson', stringifyJson(index.refs)) - .setProperty(index.id, 'recentCaptureRecordsJson', stringifyJson(index.records)) - .setProperty(index.id, 'fastCaptureRecordsJson', stringifyJson(index.records)); -} - -function ambientScopeDescriptors(scope) { - return [ - { key: 'ambientGitRemote', value: scope.gitRemote }, - { key: 'ambientGitRoot', value: scope.gitRoot }, - { key: 'ambientCwd', value: scope.cwd }, - ].filter((descriptor) => descriptor.value); -} - -function parseRefs(value) { - if (!value) { - return []; - } - - try { - const parsed = parseJson(value); - if (!Array.isArray(parsed)) { - return []; - } - - return parsed - .map(normalizeRefRecord) - .filter(Boolean) - .sort(compareEntriesNewestFirst); - } catch { - return []; - } -} - -function parseRecords(value) { - if (!value) { - return []; - } - - try { - const parsed = parseJson(value); - if (!Array.isArray(parsed)) { - return []; - } - - return parsed - .map(normalizeRecordRecord) - .filter(Boolean) - .sort(compareEntriesNewestFirst); - } catch { - return []; - } -} - -function parseRef(value) { - if (!value) { - return null; - } - - try { - return normalizeRefRecord(parseJson(value)); - } catch { - return null; - } -} - -function parseRecord(value) { - if (!value) { - return null; - } - - try { - return normalizeRecordRecord(parseJson(value)); - } catch { - return null; - } -} - -function normalizeRefRecord(record) { - if (!record?.id) { - return null; - } - - return Object.freeze({ - id: String(record.id), - createdAt: record.createdAt ? String(record.createdAt) : null, - sortKey: String(record.sortKey ?? ''), - }); -} - -function normalizeRecordRecord(record) { - const ref = normalizeRefRecord(record); - if (!ref) { - return null; - } - - return Object.freeze({ - ...ref, - kind: 'capture', - writerId: stringOrNull(record.writerId), - text: String(record.text ?? ''), - source: stringOrNull(record.source), - channel: stringOrNull(record.channel), - thoughtId: stringOrNull(record.thoughtId), - sessionId: stringOrNull(record.sessionId), - ambientCwd: stringOrNull(record.ambientCwd), - ambientGitRoot: stringOrNull(record.ambientGitRoot), - ambientGitRemote: stringOrNull(record.ambientGitRemote), - ambientGitBranch: stringOrNull(record.ambientGitBranch), - captureIngress: stringOrNull(record.captureIngress), - captureSourceApp: stringOrNull(record.captureSourceApp), - captureSourceURL: stringOrNull(record.captureSourceURL), - }); -} - -function captureRecordToRef(record) { - return Object.freeze({ - id: record.id, - createdAt: record.createdAt, - sortKey: record.sortKey, - }); -} - -function mergeRecords(records, record, limit) { - const byId = new Map(records.map((candidate) => [candidate.id, candidate])); - if (record) { - byId.set(record.id, record); - } - return [...byId.values()] - .sort(compareEntriesNewestFirst) - .slice(0, limit); -} - -function assignStringFields(target, source, fields) { - for (const field of fields) { - target[field] = stringOrNull(source[field]); - } -} - -function assignAmbientFields(target, entry, ambientContext) { - for (const [recordField, contextField] of CAPTURE_RECORD_AMBIENT_FIELDS) { - target[recordField] = firstString([ - objectField(ambientContext, contextField), - entry[recordField], - ]); - } -} - -function assignProvenanceFields(target, entry) { - for (const [recordField, provenanceField] of CAPTURE_RECORD_PROVENANCE_FIELDS) { - target[recordField] = firstString([ - objectField(entry.captureProvenance, provenanceField), - entry[recordField], - ]); - } -} - -function firstString(values) { - for (const value of values) { - const normalized = stringOrNull(value); - if (normalized !== null) { - return normalized; - } - } - - return null; -} - -function objectField(value, key) { - if (!value || typeof value !== 'object') { - return null; - } - - return value[key]; -} - -function textOrEmpty(value) { - if (value === null || value === undefined) { - return ''; - } - - return String(value); -} - -function stringOrNull(value) { - return typeof value === 'string' ? value : null; -} - -function normalizeLimit(limit, fallback) { - return Number.isInteger(limit) ? Math.max(0, limit) : fallback; -} diff --git a/src/store/reflect.js b/src/store/reflect.js index 94edcecb..0b5bd32e 100644 --- a/src/store/reflect.js +++ b/src/store/reflect.js @@ -3,9 +3,7 @@ import { CONSTRAINT_PROMPTS, MAX_REFLECT_STEPS, SHARPEN_PROMPTS, - TEXT_MIME, } from './constants.js'; -import { encodeTextContent } from './content.js'; import { createEntry, createReflectSession, @@ -13,25 +11,29 @@ import { stableHash, } from './model.js'; import { - commitThinkWorldline, + appendIndexedMemoryObject, + updateIndexedMemoryObject, +} from './native-index.js'; +import { openNativeMemory } from './native-runtime.js'; +import { getReflectSession, getStoredEntry, openProductReadHandle, - openThinkWorldline, } from './runtime.js'; import { assessReflectability } from './derivation.js'; -export async function startReflect(repoDir, seedEntryId, { promptType = null } = {}) { - const worldline = await openThinkWorldline(repoDir); +export async function startReflect(repoDir, seedEntryId, { + promptType = null, +} = {}) { + const memory = await openNativeMemory(repoDir); const read = await openProductReadHandle(repoDir); const planned = await planReflect(read, seedEntryId, { promptType }); - if (!planned.ok) { return planned; } - const {promptPlan} = planned; - const session = createReflectSession(worldline.writerId, { + const { promptPlan } = planned; + const session = createReflectSession(memory.writerId, { seedEntryId, contrastEntryId: null, promptType: promptPlan.promptType, @@ -39,29 +41,25 @@ export async function startReflect(repoDir, seedEntryId, { promptType = null } = selectionReason: promptPlan.selectionReason, }); - // eslint-disable-next-line require-await -- git-warp patch callback must be async for the library API - await commitThinkWorldline(repoDir, async patch => { - patch - .addNode(session.id) - .setProperty(session.id, 'kind', session.kind) - .setProperty(session.id, 'source', session.source) - .setProperty(session.id, 'channel', session.channel) - .setProperty(session.id, 'writerId', session.writerId) - .setProperty(session.id, 'createdAt', session.createdAt) - .setProperty(session.id, 'sortKey', session.sortKey) - .setProperty(session.id, 'seedEntryId', session.seedEntryId) - .setProperty(session.id, 'promptType', session.promptType) - .setProperty(session.id, 'question', session.question) - .setProperty(session.id, 'selectionReasonKind', session.selectionReason.kind) - .setProperty(session.id, 'selectionReasonText', session.selectionReason.text) - .setProperty(session.id, 'maxSteps', session.maxSteps) - .setProperty(session.id, 'stepCount', 0); - - patch.addEdge(session.id, session.seedEntryId, 'seeded_by'); - - if (session.contrastEntryId) { - patch.setProperty(session.id, 'contrastEntryId', session.contrastEntryId); - } + await appendIndexedMemoryObject(repoDir, { + id: session.id, + kind: session.kind, + facts: { + kind: session.kind, + source: session.source, + channel: session.channel, + writerId: session.writerId, + createdAt: session.createdAt, + sortKey: session.sortKey, + seedEntryId: session.seedEntryId, + contrastEntryId: session.contrastEntryId, + promptType: session.promptType, + question: session.question, + selectionReasonKind: session.selectionReason.kind, + selectionReasonText: session.selectionReason.text, + maxSteps: session.maxSteps, + stepCount: 0, + }, }); return Object.freeze({ @@ -78,14 +76,14 @@ export async function startReflect(repoDir, seedEntryId, { promptType = null } = }); } -export async function previewReflect(repoDir, seedEntryId, { promptType = null } = {}) { +export async function previewReflect(repoDir, seedEntryId, { + promptType = null, +} = {}) { const read = await openProductReadHandle(repoDir); const planned = await planReflect(read, seedEntryId, { promptType }); - if (!planned.ok) { return planned; } - return Object.freeze({ ok: true, seedEntryId, @@ -100,15 +98,14 @@ export async function previewReflect(repoDir, seedEntryId, { promptType = null } } export async function saveReflectResponse(repoDir, sessionId, response) { - const worldline = await openThinkWorldline(repoDir); + const memory = await openNativeMemory(repoDir); const read = await openProductReadHandle(repoDir); const session = await getReflectSession(read, sessionId); - if (!session) { return null; } - const entry = createEntry(response, worldline.writerId, { + const entry = createEntry(response, memory.writerId, { kind: 'reflect', source: 'reflect', seedEntryId: session.seedEntryId, @@ -117,32 +114,30 @@ export async function saveReflectResponse(repoDir, sessionId, response) { promptType: session.promptType, }); - await commitThinkWorldline(repoDir, async patch => { - patch - .addNode(entry.id) - .setProperty(entry.id, 'kind', entry.kind) - .setProperty(entry.id, 'source', entry.source) - .setProperty(entry.id, 'channel', entry.channel) - .setProperty(entry.id, 'writerId', entry.writerId) - .setProperty(entry.id, 'createdAt', entry.createdAt) - .setProperty(entry.id, 'sortKey', entry.sortKey) - .setProperty(entry.id, 'seedEntryId', entry.seedEntryId) - .setProperty(entry.id, 'sessionId', entry.sessionId) - .setProperty(entry.id, 'promptType', entry.promptType); - - patch - .addEdge(entry.id, entry.sessionId, 'produced_in') - .addEdge(entry.id, entry.seedEntryId, 'responds_to'); - - if (entry.contrastEntryId) { - patch.setProperty(entry.id, 'contrastEntryId', entry.contrastEntryId); - } - - patch - .setProperty(session.id, 'stepCount', session.stepCount + 1) - .setProperty(session.id, 'updatedAt', entry.createdAt); - - await patch.attachContent(entry.id, encodeTextContent(response), { mime: TEXT_MIME }); + await appendIndexedMemoryObject(repoDir, { + id: entry.id, + kind: entry.kind, + facts: { + kind: entry.kind, + text: entry.text, + source: entry.source, + channel: entry.channel, + writerId: entry.writerId, + createdAt: entry.createdAt, + sortKey: entry.sortKey, + seedEntryId: entry.seedEntryId, + contrastEntryId: entry.contrastEntryId, + sessionId: entry.sessionId, + promptType: entry.promptType, + }, + }); + await updateIndexedMemoryObject(repoDir, { + id: session.id, + kind: session.kind, + facts: { + stepCount: session.stepCount + 1, + updatedAt: entry.createdAt, + }, }); return entry; @@ -150,73 +145,27 @@ export async function saveReflectResponse(repoDir, sessionId, response) { function selectReflectPrompt(seedEntry, requestedPromptType = null) { const normalized = normalizeSeed(seedEntry.text); - if (requestedPromptType === 'challenge') { - return Object.freeze({ - promptType: 'challenge', - selectionReason: Object.freeze({ - kind: 'requested_challenge', - text: 'Used the requested challenge prompt family for this reflect session.', - }), - question: pickDeterministicPrompt(CHALLENGE_PROMPTS, normalized), - }); + return buildPromptPlan('challenge', 'requested_challenge', CHALLENGE_PROMPTS, normalized); } - if (requestedPromptType === 'constraint') { - return Object.freeze({ - promptType: 'constraint', - selectionReason: Object.freeze({ - kind: 'requested_constraint', - text: 'Used the requested constraint prompt family for this reflect session.', - }), - question: pickDeterministicPrompt(CONSTRAINT_PROMPTS, normalized), - }); + return buildPromptPlan('constraint', 'requested_constraint', CONSTRAINT_PROMPTS, normalized); } - if (requestedPromptType === 'sharpen') { - return Object.freeze({ - promptType: 'sharpen', - selectionReason: Object.freeze({ - kind: 'requested_sharpen', - text: 'Used the requested sharpen prompt family for this reflect session.', - }), - question: pickDeterministicPrompt(SHARPEN_PROMPTS, normalized), - }); + return buildPromptPlan('sharpen', 'requested_sharpen', SHARPEN_PROMPTS, normalized); } - - const familyIndex = stableHash(normalized) % 2; - - if (familyIndex === 0) { - return Object.freeze({ - promptType: 'challenge', - selectionReason: Object.freeze({ - kind: 'seed_only_challenge', - text: 'Used a deterministic challenge prompt from the seed thought alone.', - }), - question: pickDeterministicPrompt(CHALLENGE_PROMPTS, normalized), - }); - } - - return Object.freeze({ - promptType: 'constraint', - selectionReason: Object.freeze({ - kind: 'seed_only_constraint', - text: 'Used a deterministic constraint prompt from the seed thought alone.', - }), - question: pickDeterministicPrompt(CONSTRAINT_PROMPTS, normalized), - }); + return stableHash(normalized) % 2 === 0 + ? buildPromptPlan('challenge', 'seed_only_challenge', CHALLENGE_PROMPTS, normalized) + : buildPromptPlan('constraint', 'seed_only_constraint', CONSTRAINT_PROMPTS, normalized); } -async function planReflect(read, seedEntryId, { promptType = null } = {}) { +async function planReflect(read, seedEntryId, { + promptType = null, +} = {}) { const seedEntry = await getStoredEntry(read, seedEntryId); - if (!seedEntry || seedEntry.kind !== 'capture') { - return Object.freeze({ - ok: false, - code: 'seed_not_found', - }); + return Object.freeze({ ok: false, code: 'seed_not_found' }); } - const eligibility = assessReflectability(seedEntry.text); if (!eligibility.eligible) { return Object.freeze({ @@ -227,7 +176,6 @@ async function planReflect(read, seedEntryId, { promptType = null } = {}) { eligibility, }); } - return Object.freeze({ ok: true, seedEntry, @@ -235,7 +183,19 @@ async function planReflect(read, seedEntryId, { promptType = null } = {}) { }); } -function pickDeterministicPrompt(prompts, normalizedSeed) { - const index = stableHash(normalizedSeed) % prompts.length; - return prompts[index]; +function buildPromptPlan(promptType, reasonKind, prompts, normalizedSeed) { + return Object.freeze({ + promptType, + selectionReason: Object.freeze({ + kind: reasonKind, + text: selectionReasonText(promptType, reasonKind), + }), + question: prompts[stableHash(normalizedSeed) % prompts.length], + }); +} + +function selectionReasonText(promptType, reasonKind) { + return reasonKind.startsWith('requested_') + ? `Used the requested ${promptType} prompt family for this reflect session.` + : `Used a deterministic ${promptType} prompt from the seed thought alone.`; } diff --git a/src/store/runtime.js b/src/store/runtime.js index fed0af60..1986c843 100644 --- a/src/store/runtime.js +++ b/src/store/runtime.js @@ -1,53 +1,110 @@ -import { GitGraphAdapter, openWarpWorldline } from '@git-stunts/git-warp'; - -import { ContentUnavailableError } from '../errors.js'; import { resolveHistorySessionEntries } from '../history/session.js'; -import { createThinkPlumbing } from '../git.js'; import { - ARTIFACT_PREFIX, - ENTRY_PREFIX, - EXACT_READ_UNAVAILABLE, GRAPH_META_ID, GRAPH_MODEL_VERSION, - GRAPH_NAME, - LEGACY_BRAINSTORM_SESSION_PREFIX, - PRODUCT_READ_LENS, - REFLECT_SESSION_PREFIX, SESSION_KINDS, - SESSION_PREFIX, - THOUGHT_PREFIX, } from './constants.js'; +import { compareEntriesNewestFirst } from './model.js'; +import { + listIndexedMemoryDocuments, + readIndexedMemoryDocument, + readNativeIndexSummary, +} from './native-index.js'; import { - compareEntriesNewestFirst, - createWriterId, - storesTextContent, -} from './model.js'; + decodeNativeDocument, +} from './native-document.js'; import { - captureRecordToProps, - readCaptureReadModel, - readCaptureRecords, -} from './read-model.js'; + closeNativeMemory, + openNativeMemory, +} from './native-runtime.js'; + +const COMMON_ENTRY_FACTS = Object.freeze([ + 'kind', + 'text', + 'writerId', + 'createdAt', + 'sortKey', + 'source', + 'channel', +]); +const CAPTURE_FACTS = Object.freeze([ + 'thoughtId', + 'sessionId', + 'ambientCwd', + 'ambientGitRoot', + 'ambientGitRemote', + 'ambientGitBranch', + 'captureIngress', + 'captureSourceApp', + 'captureSourceURL', +]); +const REFLECT_FACTS = Object.freeze([ + 'seedEntryId', + 'contrastEntryId', + 'sessionId', + 'promptType', + 'question', + 'selectionReasonKind', + 'selectionReasonText', + 'stepCount', + 'maxSteps', +]); +const ANNOTATION_FACTS = Object.freeze([ + 'targetEntryId', +]); +const DOMAIN_FACTS = Object.freeze([ + 'name', + 'normalizedName', + 'thoughtCount', + 'primaryInputKind', + 'primaryInputId', + 'verdict', + 'reasonKind', + 'reasonText', + 'deriver', + 'deriverVersion', + 'schemaVersion', + 'updatedAt', +]); +const ENTRY_FACTS = Object.freeze([ + ...COMMON_ENTRY_FACTS, + ...CAPTURE_FACTS, + ...REFLECT_FACTS, + ...ANNOTATION_FACTS, + ...DOMAIN_FACTS, +]); +const NO_INDEXED_KINDS = Object.freeze([]); +const INDEXED_KINDS_BY_PREFIX = Object.freeze([ + ['entry:', Object.freeze(['capture', 'reflect'])], + ['session:', Object.freeze(['session'])], + ['reflect:', Object.freeze(['reflect_session'])], + ['brainstorm:', Object.freeze(['brainstorm_session'])], + ['annotation:', Object.freeze(['annotation'])], + ['artifact:', Object.freeze(['auto_tags', 'semantic_parse'])], + ['keyword:', Object.freeze(['keyword'])], + ['topic:', Object.freeze(['topic'])], + ['classification:', Object.freeze(['classification'])], + ['thought:', Object.freeze(['thought'])], + ['evolution:', Object.freeze(['evolution'])], +]); export class GenericEntry { constructor(nodeId, resolvedProps, text) { + Object.assign(this, resolvedProps); this.id = nodeId; this.kind = resolvedProps.kind; this.writerId = resolvedProps.writerId; this.createdAt = resolvedProps.createdAt; this.sortKey = String(resolvedProps.sortKey || ''); this.text = text; - Object.freeze(this); + this.source = resolvedProps.source ?? null; + this.channel = resolvedProps.channel ?? null; } } -export class CaptureEntry { +export class CaptureEntry extends GenericEntry { constructor(nodeId, resolvedProps, text) { - this.id = nodeId; - this.kind = resolvedProps.kind; - this.writerId = resolvedProps.writerId; - this.createdAt = resolvedProps.createdAt; - this.sortKey = String(resolvedProps.sortKey || ''); - this.text = text; + super(nodeId, resolvedProps, text); this.source = resolvedProps.source; this.channel = resolvedProps.channel; this.thoughtId = resolvedProps.thoughtId ?? null; @@ -56,27 +113,17 @@ export class CaptureEntry { this.ambientGitRoot = resolvedProps.ambientGitRoot ?? null; this.ambientGitRemote = resolvedProps.ambientGitRemote ?? null; this.ambientGitBranch = resolvedProps.ambientGitBranch ?? null; - this.captureProvenance = resolvedProps.captureIngress || resolvedProps.captureSourceApp || resolvedProps.captureSourceURL - ? Object.freeze({ - ingress: resolvedProps.captureIngress ?? null, - sourceApp: resolvedProps.captureSourceApp ?? null, - sourceURL: resolvedProps.captureSourceURL ?? null, - }) - : null; + this.captureProvenance = captureProvenanceFrom(resolvedProps); Object.freeze(this); } } -export class ReflectEntry { +export class ReflectEntry extends GenericEntry { constructor(nodeId, resolvedProps, text) { - this.id = nodeId; - this.kind = resolvedProps.kind; - this.writerId = resolvedProps.writerId; - this.createdAt = resolvedProps.createdAt; - this.sortKey = String(resolvedProps.sortKey || ''); - this.text = text; + super(nodeId, resolvedProps, text); this.seedEntryId = resolvedProps.seedEntryId ?? null; this.contrastEntryId = resolvedProps.contrastEntryId ?? null; + this.sessionId = resolvedProps.sessionId ?? null; this.promptType = resolvedProps.promptType ?? null; this.question = resolvedProps.question ?? null; this.selectionReason = resolvedProps.selectionReasonKind @@ -91,658 +138,283 @@ export class ReflectEntry { } } -export class AnnotationEntry { +export class AnnotationEntry extends GenericEntry { constructor(nodeId, resolvedProps, text) { - this.id = nodeId; - this.kind = resolvedProps.kind; - this.writerId = resolvedProps.writerId; - this.createdAt = resolvedProps.createdAt; - this.sortKey = String(resolvedProps.sortKey || ''); - this.text = text; + super(nodeId, resolvedProps, text); + this.targetEntryId = resolvedProps.targetEntryId ?? null; Object.freeze(this); } } export class BaseEntry { static from(nodeId, resolvedProps, text) { - if (resolvedProps.kind === 'capture') { return new CaptureEntry(nodeId, resolvedProps, text); } + if (resolvedProps.kind === 'capture') { + return new CaptureEntry(nodeId, resolvedProps, text); + } if (resolvedProps.kind === 'reflect' || SESSION_KINDS.includes(resolvedProps.kind)) { return new ReflectEntry(nodeId, resolvedProps, text); } - if (resolvedProps.kind === 'annotation') { return new AnnotationEntry(nodeId, resolvedProps, text); } - return new GenericEntry(nodeId, resolvedProps, text); - } -} - -const WRITER_CAS_CONFLICT_TEXT = 'writer ref was updated by another process'; -const DEFAULT_PATCH_MAX_ATTEMPTS = 3; -const warpWorldlineCache = new Map(); -const runtimeBlobStorageCache = new Map(); -const opticBasisCache = new WeakMap(); - -export async function openThinkWorldline(repoDir) { - const cached = warpWorldlineCache.get(repoDir); - if (cached) { - return cached; - } - - const worldline = await openThinkWorldlineOnce(repoDir, createWriterId()); - warpWorldlineCache.set(repoDir, worldline); - return worldline; -} - -async function openThinkWorldlineOnce(repoDir, writerId) { - const persistence = createThinkWarpPersistence(repoDir); - - return await openWarpWorldline({ - persistence, - worldlineName: GRAPH_NAME, - writerId, - }); -} - -function createThinkWarpPersistence(repoDir) { - return new GitGraphAdapter({ - plumbing: createThinkPlumbing(repoDir), - }); -} - -export function clearWarpRuntimeCache(repoDir) { - warpWorldlineCache.delete(repoDir); - runtimeBlobStorageCache.delete(repoDir); -} - -export async function commitThinkWorldline(repoDir, patcher, { - maxAttempts = DEFAULT_PATCH_MAX_ATTEMPTS, -} = {}) { - let attempt = 1; - - /* eslint-disable no-await-in-loop -- retry attempts must run sequentially against a refreshed worldline */ - while (true) { - const worldline = await openThinkWorldline(repoDir); - - try { - await worldline.commit(patcher); - return worldline; - } catch (error) { - if (!isWriterCasConflict(error) || attempt >= maxAttempts) { - throw error; - } - - clearWarpRuntimeCache(repoDir); - attempt += 1; - } - } - /* eslint-enable no-await-in-loop */ -} - -export async function commitThinkWorldlineWithWriter(repoDir, writerId, patcher, { - maxAttempts = DEFAULT_PATCH_MAX_ATTEMPTS, -} = {}) { - let attempt = 1; - - /* eslint-disable no-await-in-loop -- retry attempts must run sequentially against a refreshed worldline */ - while (true) { - const worldline = await openThinkWorldlineOnce(repoDir, writerId); - - try { - await worldline.commit(patcher); - return worldline; - } catch (error) { - if (!isWriterCasConflict(error) || attempt >= maxAttempts) { - throw error; - } - - attempt += 1; + if (resolvedProps.kind === 'annotation') { + return new AnnotationEntry(nodeId, resolvedProps, text); } + return Object.freeze(new GenericEntry(nodeId, resolvedProps, text)); } - /* eslint-enable no-await-in-loop */ -} - -export function isWriterCasConflict(error) { - return error instanceof Error && error.message.includes(WRITER_CAS_CONFLICT_TEXT); } export async function openProductReadHandle(repoDir) { - return await createWorldlineProductReadHandle({ repoDir }); -} - -async function createWorldlineProductReadHandle({ - repoDir, -}) { - const worldline = await openThinkWorldline(repoDir); - const blobStorage = await getRuntimeBlobStorage(repoDir); - - return { - app: null, + const memory = await openNativeMemory(repoDir); + return Object.freeze({ repoDir, - worldline, - view: worldline.live(), - contentCore: null, - blobStorage, - readContent: readUnavailableRuntimeContent, - readContentRequiresContentOid: true, - readNodeProp: createWorldlineExactPropReader(worldline), - writerId: worldline.writerId, - }; -} - -function readUnavailableRuntimeContent(nodeId) { - throw new ContentUnavailableError( - `Content for ${nodeId} is unavailable: this git-warp runtime exposes no public content reader fallback.`, - ); -} - -function createWorldlineExactPropReader(worldline) { - return async (nodeId, key) => { - if (opticBasisCache.get(worldline) === false) { - return EXACT_READ_UNAVAILABLE; - } - - try { - await prepareCachedOpticBasis(worldline); - - const result = await worldline.optic().node(nodeId).prop(key).read(); - return result.exists ? result.value : undefined; - } catch (error) { - if (isMissingBoundedOpticBasis(error)) { - return EXACT_READ_UNAVAILABLE; - } - throw error; - } - }; -} - -async function prepareCachedOpticBasis(worldline) { - const cached = opticBasisCache.get(worldline); - if (cached) { - return cached; - } - - const basisPromise = worldline.prepareOpticBasis(); - opticBasisCache.set(worldline, basisPromise); - - try { - await basisPromise; - return basisPromise; - } catch (error) { - if (isMissingBoundedOpticBasis(error)) { - opticBasisCache.set(worldline, false); - } else { - opticBasisCache.delete(worldline); - } - throw error; - } -} - -function isMissingBoundedOpticBasis(error) { - return error instanceof Error && ( - error.code === 'E_OPTIC_NO_BOUNDED_BASIS' || - error.message.includes('E_OPTIC_NO_BOUNDED_BASIS') - ); -} - -async function getRuntimeBlobStorage(repoDir) { - if (runtimeBlobStorageCache.has(repoDir)) { - return await runtimeBlobStorageCache.get(repoDir); - } - - const plumbing = createThinkPlumbing(repoDir); - const persistence = new GitGraphAdapter({ plumbing }); - const blobStorage = createRuntimeBlobStorage(persistence); - runtimeBlobStorageCache.set(repoDir, blobStorage); - return await blobStorage; + memory, + writerId: memory.writerId, + readDocument: nodeId => readIndexedMemoryDocument(repoDir, nodeId, { + kinds: indexedKindsForId(nodeId), + memory, + }), + listDocumentsByKind: (kind, options) => + listIndexedMemoryDocuments(repoDir, kind, { + ...options, + memory, + }), + }); } -function createRuntimeBlobStorage(persistence) { - const createStorage = persistence.createRuntimeBlobStorage; - if (typeof createStorage !== 'function') { - return null; - } - return createStorage.call(persistence); +export async function clearWarpRuntimeCache(repoDir) { + await closeNativeMemory(repoDir); } export async function getGraphModelStatusForRead(read) { - const props = await read.view.getNodeProps(GRAPH_META_ID); - const currentGraphModelVersion = Number(props?.graphModelVersion ?? 1); - - return { + const document = decodeNativeDocument( + await read.memory.memoryDocument(GRAPH_META_ID), + 'memory-object' + ); + const currentGraphModelVersion = Number( + document?.graphModelVersion ?? GRAPH_MODEL_VERSION + ); + return Object.freeze({ currentGraphModelVersion, requiredGraphModelVersion: GRAPH_MODEL_VERSION, migrationRequired: currentGraphModelVersion < GRAPH_MODEL_VERSION, - }; + }); } export async function getStoredEntry(read, nodeId, props = null) { - const resolvedProps = props ?? await read.view.getNodeProps(nodeId); - if (!resolvedProps) { + const document = props ?? await read.readDocument(nodeId); + if (!document?.kind) { return null; } + return BaseEntry.from(nodeId, document, document.text ?? ''); +} - const text = storesTextContent(resolvedProps.kind) - ? await readNodeText(read, nodeId, resolvedProps) - : ''; - - return BaseEntry.from(nodeId, resolvedProps, text); +function indexedKindsForId(nodeId) { + return INDEXED_KINDS_BY_PREFIX + .find(([prefix]) => nodeId.startsWith(prefix))?.[1] + ?? NO_INDEXED_KINDS; } export function toBrowseEntry(entry) { if (!entry) { return null; } - - return { + return Object.freeze({ id: entry.id, text: entry.text, sortKey: entry.sortKey, createdAt: entry.createdAt, sessionId: entry.sessionId ?? null, - }; + }); } export async function getReflectSession(read, sessionId) { const session = await getStoredEntry(read, sessionId); - if (!session || !SESSION_KINDS.includes(session.kind)) { - return null; - } - - return session; + return session && SESSION_KINDS.includes(session.kind) ? session : null; } -export async function listEntriesByKind(read, kind) { - const result = await read.view.query() - .match(getMatchPatternsForKind(kind)) - .where({ kind }) - .run(); - - const entries = []; - for (const node of result.nodes ?? []) { - // eslint-disable-next-line no-await-in-loop -- sequential graph reads per query result node - const entry = await getStoredEntry(read, node.id, node.props ?? null); - if (entry) { - entries.push(entry); - } - } - - return entries; +export async function listEntriesByKind(read, kind, { + limit = 500, +} = {}) { + const documents = await read.listDocumentsByKind(kind, { limit }); + return documents + .filter(document => document.kind === kind) + .map(document => BaseEntry.from(document.id, document, document.text ?? '')); } -export async function listEntryPropsByKind(read, kind) { - const result = await read.view.query() - .match(getMatchPatternsForKind(kind)) - .where({ kind }) - .run(); - - return (result.nodes ?? []).map((node) => Object.freeze({ - id: node.id, - ...(node.props ?? {}), +export async function listEntryPropsByKind(read, kind, { + limit = 500, +} = {}) { + const entries = await listEntriesByKind(read, kind, { limit }); + return entries.map(entry => Object.freeze({ + id: entry.id, + ...entryProps(entry), })); } -function getMatchPatternsForKind(kind) { - if (kind === 'capture' || kind === 'reflect') { - return `${ENTRY_PREFIX}*`; - } - if (kind === 'reflect_session' || kind === 'brainstorm_session') { - return [`${REFLECT_SESSION_PREFIX}*`, `${LEGACY_BRAINSTORM_SESSION_PREFIX}*`]; - } - if (kind === 'session') { - return `${SESSION_PREFIX}*`; - } - if (kind === 'seed_quality' || kind === 'session_attribution') { - return `${ARTIFACT_PREFIX}*`; - } - if (kind === 'thought') { - return `${THOUGHT_PREFIX}*`; - } - if (kind === 'graph_meta') { - return GRAPH_META_ID; - } - return PRODUCT_READ_LENS.match; -} - export async function listChronologyEntries(read) { - const captures = await listRecentStoredEntries(read, { + const entries = await listRecentStoredEntries(read, { kind: 'capture', - limit: Number.MAX_SAFE_INTEGER, + limit: 4096, }); - return captures - .map(toBrowseEntry) - .sort(compareEntriesNewestFirst); + return entries.map(toBrowseEntry); } export async function getSingleNeighborId(read, nodeId, direction, label) { - const query = read.view.query().match(nodeId); - const result = direction === 'incoming' - ? await query.incoming(label).run() - : await query.outgoing(label).run(); - return result.nodes?.[0]?.id ?? null; -} - -export async function getLatestStoredEntry(read, kind = 'capture', options = {}) { - const latestId = await getLatestIdByKind(read, kind, options); - return latestId ? await getStoredEntry(read, latestId) : null; -} - -export async function listRecentStoredEntries(read, { kind = 'capture', limit = 50 } = {}) { - const entries = []; - for await (const entry of iterateRecentStoredEntries(read, { kind, limit })) { - entries.push(entry); - } - return entries; -} - -export async function* iterateRecentStoredEntries(read, { kind = 'capture', limit = 50 } = {}) { - const maxEntries = Number.isInteger(limit) ? Math.max(0, limit) : 50; - if (maxEntries === 0) { - return; - } - - for (const entry of await listRecentCandidateEntries(read, { kind, limit: maxEntries })) { - yield entry; - } -} - -async function getLatestIdByKind(read, kind, options = {}) { - if (kind !== 'capture') { + if (direction !== 'outgoing') { return null; } - - return await getLatestCaptureId(read, options); + const document = await read.readDocument(nodeId); + const field = Object.freeze({ + annotates: 'targetEntryId', + derived_from: 'primaryInputId', + produced_in: 'sessionId', + responds_to: 'seedEntryId', + seeded_by: 'seedEntryId', + })[label]; + return field ? document?.[field] ?? null : null; } -export async function readNodeText(read, nodeId, props = null) { - const contentOid = await resolveNodeContentOid(read, nodeId, props); - const content = await readNodeContent(read, nodeId, contentOid); - if (hasReadableContent(content)) { - return decodeContent(content); - } - - if (contentOid) { - throw new ContentUnavailableError( - `Content for ${nodeId} is unavailable: the read handle has a content oid but no readable content source.`, - ); - } - - return ''; -} - -async function resolveNodeContentOid(read, nodeId, props) { - const resolvedProps = props ?? await read.view.getNodeProps(nodeId); - if (typeof resolvedProps?._content === 'string') { - return resolvedProps._content; - } - return await readNodeContentOid(read, nodeId); -} - -async function readNodeContent(read, nodeId, contentOid) { - const attachedContent = await readAttachedContent(read, contentOid); - if (hasReadableContent(attachedContent)) { - return attachedContent; - } - if (!contentOid && read.readContentRequiresContentOid) { - return null; - } - return await readContent(read, nodeId); -} - -async function readAttachedContent(read, contentOid) { - if (contentOid && read.blobStorage) { - return await read.blobStorage.retrieve(contentOid); - } - return null; -} - -function decodeContent(content) { - return new TextDecoder().decode(content); +export async function getLatestStoredEntry(read, kind = 'capture', { + excludeIds = [], +} = {}) { + const entries = await listEntriesByKind(read, kind, { + limit: excludeIds.length + 1, + }); + const excluded = new Set(excludeIds); + return entries.find(entry => !excluded.has(entry.id)) ?? null; } -function hasReadableContent(content) { - return content !== null && content !== undefined; +export async function listRecentStoredEntries(read, { + kind = 'capture', + limit = 50, +} = {}) { + return await listEntriesByKind(read, kind, { limit }); } -async function readContent(read, nodeId) { - if (typeof read.readContent === 'function') { - return await read.readContent(nodeId); +export async function* iterateRecentStoredEntries(read, options = {}) { + for (const entry of await listRecentStoredEntries(read, options)) { + yield entry; } - return await read.contentCore?.getContent?.(nodeId) ?? null; } -async function readNodeContentOid(read, nodeId) { - if (typeof read.view.getNodeContentMeta !== 'function') { - return null; +export async function readNodeText(read, nodeId, props = null) { + if (typeof props?.text === 'string') { + return props.text; } - const contentMeta = await read.view.getNodeContentMeta(nodeId); - return typeof contentMeta?.oid === 'string' ? contentMeta.oid : null; + const entry = await getStoredEntry(read, nodeId); + return entry?.text ?? ''; } -export async function getLatestCaptureId(read, { excludeIds = [] } = {}) { - const index = await readCaptureReadModel(read); - const excludedIds = new Set(excludeIds); - return index.refs.find((ref) => !excludedIds.has(ref.id))?.id ?? null; +export async function getLatestCaptureId(read, { + excludeIds = [], +} = {}) { + const excluded = new Set(excludeIds); + const entries = await listEntriesByKind(read, 'capture', { + limit: excludeIds.length + 1, + }); + return entries.find(entry => !excluded.has(entry.id))?.id ?? null; } -export async function getProducedInSessionId(read, entry) { - const result = await read.view.query() - .match(entry.id) - .outgoing('produced_in') - .run(); - return result.nodes?.[0]?.id ?? entry.sessionId ?? null; +export function getProducedInSessionId(_read, entry) { + return entry.sessionId ?? null; } -// eslint-disable-next-line require-await -- wraps git-warp view.hasNode which returns a promise export async function hasNode(read, nodeId) { - return read.view.hasNode(nodeId); + return await read.readDocument(nodeId) !== null; } export async function resolveHistorySessionTraversal(read, entry) { - if (!entry) { + if (!entry?.sessionId) { return emptySessionTraversal(); } - - const sessionEntries = await listHistorySessionBrowseEntries(read, entry); - const sessionIndex = sessionEntries.findIndex((candidate) => candidate.id === entry.id); - - if (sessionIndex === -1) { - return { - entries: sessionEntries, - sessionCount: sessionEntries.length, - sessionPosition: null, - previous: null, - next: null, - }; - } - - return { - entries: sessionEntries, - sessionCount: sessionEntries.length, - sessionPosition: sessionIndex + 1, - previous: sessionIndex > 0 ? sessionEntries[sessionIndex - 1] : null, - next: sessionIndex + 1 < sessionEntries.length ? sessionEntries[sessionIndex + 1] : null, - }; + const captureProps = await listIndexedCaptureProps(read, { limit: 4096 }); + const sessionProps = resolveHistorySessionEntries(captureProps, entry); + const entries = await Promise.all( + sessionProps.map(props => props.id === entry.id + ? Promise.resolve(toBrowseEntry(entry)) + : getStoredEntry(read, props.id).then(toBrowseEntry)) + ); + const visible = entries.filter(Boolean); + const sessionIndex = visible.findIndex(candidate => candidate.id === entry.id); + return Object.freeze({ + entries: visible, + sessionCount: visible.length, + sessionPosition: sessionIndex < 0 ? null : sessionIndex + 1, + previous: sessionIndex > 0 ? visible[sessionIndex - 1] : null, + next: sessionIndex >= 0 && sessionIndex + 1 < visible.length + ? visible[sessionIndex + 1] + : null, + }); } export const resolveGraphSessionTraversal = resolveHistorySessionTraversal; -async function listHistorySessionBrowseEntries(read, entry) { - if (!entry.sessionId) { - return []; - } - - const captureProps = await listIndexedCaptureProps(read, { - limit: Number.MAX_SAFE_INTEGER, +export async function listIndexedCaptureProps(read, { + limit = 50, +} = {}) { + const entries = await listRecentStoredEntries(read, { + kind: 'capture', + limit, }); - const sessionEntryProps = resolveHistorySessionEntries(captureProps, entry); - const sessionEntries = []; - - for (const props of sessionEntryProps) { - // eslint-disable-next-line no-await-in-loop -- bounded hydration of entries in the visible session - const sessionEntry = await readHistorySessionBrowseEntry(read, entry, props); - if (sessionEntry) { - sessionEntries.push(sessionEntry); - } - } - - return sessionEntries; -} - -async function readHistorySessionBrowseEntry(read, currentEntry, props) { - if (props.id === currentEntry.id) { - return toBrowseEntry(currentEntry); - } - - const capture = await getStoredEntry(read, props.id, props); - return capture?.kind === 'capture' ? toBrowseEntry(capture) : null; -} - -function emptySessionTraversal() { - return { - entries: [], - sessionCount: 0, - sessionPosition: null, - previous: null, - next: null, - }; -} - -export async function listIndexedCaptureProps(read, { limit = 50 } = {}) { - const records = await readCaptureRecords(read, { limit }); - if (records.length > 0) { - return records - .map((record) => Object.freeze({ - id: record.id, - ...captureRecordToProps(record), - })) - .sort(compareEntriesNewestFirst); - } - if (typeof read.readNodeProp === 'function') { - return []; - } - - const index = await readCaptureReadModel(read); - const maxEntries = Number.isInteger(limit) ? Math.max(0, limit) : 50; - const props = []; - - for (const ref of index.refs.slice(0, maxEntries)) { - // eslint-disable-next-line no-await-in-loop -- exact bounded read per indexed capture - const captureProps = await read.view.getNodeProps(ref.id); - if (captureProps?.kind === 'capture') { - props.push(Object.freeze({ - id: ref.id, - ...captureProps, - })); - } - } - - return props.sort(compareEntriesNewestFirst); -} - -export async function listStoredEntriesByRefs(read, refs, { limit = 50, kind = 'capture' } = {}) { - const maxEntries = Number.isInteger(limit) ? Math.max(0, limit) : 50; - const entries = await hydrateStoredEntriesByRefs(read, refs.slice(0, maxEntries), kind); - return entries.sort(compareEntriesNewestFirst); -} - -async function listRecentCandidateEntries(read, { kind, limit }) { - if (kind === 'capture') { - const recordEntries = await listRecentCaptureRecordEntries(read, { limit }); - if (recordEntries.length > 0) { - return recordEntries; - } - if (typeof read.readNodeProp === 'function') { - return []; - } - } - - const candidates = await listRecentCandidateProps(read, { kind, limit }); - const entries = []; - - for (const candidate of candidates.sort(compareEntriesNewestFirst).slice(0, limit)) { - // eslint-disable-next-line no-await-in-loop -- exact bounded read per indexed capture - const entry = await getStoredEntry(read, candidate.id, candidate); - if (entry && entry.kind === kind) { - entries.push(entry); - } - } - - return entries; + return entries.map(entry => Object.freeze({ + id: entry.id, + ...entryProps(entry), + })); } -async function listRecentCaptureRecordEntries(read, { limit }) { - const records = await readCaptureRecords(read, { limit }); - return records - .map(captureRecordToEntry) +export async function listStoredEntriesByRefs(read, refs, { + limit = 50, + kind = 'capture', +} = {}) { + const selected = refs.slice(0, Math.max(0, limit)); + const entries = await Promise.all(selected.map(ref => getStoredEntry(read, ref.id))); + return entries + .filter(entry => entry && (!kind || entry.kind === kind)) .sort(compareEntriesNewestFirst); } -function captureRecordToEntry(record) { - return new CaptureEntry(record.id, captureRecordToProps(record), record.text); -} - -async function listRecentCandidateProps(read, { kind, limit }) { - if (kind === 'capture') { - return await listIndexedCaptureProps(read, { limit }); - } - - return await listEntryPropsByKind(read, kind); -} - -async function hydrateStoredEntriesByRefs(read, refs, kind) { - const entries = []; - - for (const ref of refs) { - // eslint-disable-next-line no-await-in-loop -- exact bounded read per indexed capture - const entry = await getStoredEntry(read, ref.id); - if (isRequestedEntryKind(entry, kind)) { - entries.push(entry); - } - } - - return entries; -} - -function isRequestedEntryKind(entry, kind) { - return Boolean(entry) && (!kind || entry.kind === kind); -} - export async function getChronologyNeighborEntries(read, currentEntry) { - const indexedNeighbors = await getIndexedChronologyNeighborEntries(read, currentEntry); - if (indexedNeighbors.found) { - return Object.freeze({ - newer: indexedNeighbors.newer, - older: indexedNeighbors.older, - }); + const entries = await listRecentStoredEntries(read, { + kind: 'capture', + limit: 4096, + }); + const currentIndex = entries.findIndex(entry => entry.id === currentEntry.id); + if (currentIndex < 0) { + return Object.freeze({ newer: null, older: null }); } - return Object.freeze({ - newer: await hydrateChronologyNeighborByEdge(read, currentEntry.id, 'newer'), - older: await hydrateChronologyNeighborByEdge(read, currentEntry.id, 'older'), + newer: entries[currentIndex - 1] ?? null, + older: entries[currentIndex + 1] ?? null, }); } -async function getIndexedChronologyNeighborEntries(read, currentEntry) { - const index = await readCaptureReadModel(read); - const currentIndex = index.refs.findIndex((ref) => ref.id === currentEntry.id); - if (currentIndex < 0) { - return Object.freeze({ found: false, newer: null, older: null }); - } +export async function getIndexSummary(read, kind) { + return await readNativeIndexSummary(read.repoDir, kind); +} +function captureProvenanceFrom(props) { + if (!props.captureIngress && !props.captureSourceApp && !props.captureSourceURL) { + return null; + } return Object.freeze({ - found: true, - newer: await hydrateChronologyNeighbor(read, index.refs[currentIndex - 1] ?? null), - older: await hydrateChronologyNeighbor(read, index.refs[currentIndex + 1] ?? null), + ingress: props.captureIngress ?? null, + sourceApp: props.captureSourceApp ?? null, + sourceURL: props.captureSourceURL ?? null, }); } -async function hydrateChronologyNeighbor(read, ref) { - return ref ? await getStoredEntry(read, ref.id) : null; +function entryProps(entry) { + return Object.freeze(Object.fromEntries( + ENTRY_FACTS + .filter(key => key !== 'text' && entry[key] !== null && entry[key] !== undefined) + .map(key => [key, entry[key]]) + )); } -async function hydrateChronologyNeighborByEdge(read, entryId, label) { - const result = await read.view.query() - .match(entryId) - .outgoing(label) - .run(); - const neighborId = result.nodes?.[0]?.id ?? null; - return neighborId ? await getStoredEntry(read, neighborId) : null; +function emptySessionTraversal() { + return Object.freeze({ + entries: [], + sessionCount: 0, + sessionPosition: null, + previous: null, + next: null, + }); } diff --git a/test/acceptance/mcp.test.js b/test/acceptance/mcp.test.js index fb7c0054..d73b5d90 100644 --- a/test/acceptance/mcp.test.js +++ b/test/acceptance/mcp.test.js @@ -36,42 +36,10 @@ test('think MCP server lists the core Think tools', async () => { test('think MCP capture, recent, browse, and inspect route through the existing Think runtime', async () => { const context = await createThinkContext(); - await withThinkMcpClient(context, async ({ client }) => { - const capture = await callTool(client, 'capture', { - text: 'MCP capture should feel like the same capture core, not a second product.', - }); - - assert.equal(capture.status, 'saved_locally', 'Expected MCP capture to report the normal saved-locally status.'); - assert.equal(capture.backupStatus, 'skipped', 'Expected MCP capture to preserve the normal no-upstream backup behavior.'); - assert.equal(capture.warnings.length, 0, 'Expected MCP capture not to emit post-save warnings on a healthy repo.'); - - const recent = await callTool(client, 'recent', {}); - assert.equal(recent.repoPresent, true, 'Expected recent to see the repo bootstrapped by MCP capture.'); - assert.equal(recent.entries.length, 1, 'Expected one recent entry after the first MCP capture.'); - assert.equal( - recent.entries[0].text, - 'MCP capture should feel like the same capture core, not a second product.', - 'Expected recent to surface the exact MCP-captured text.' - ); - - const browse = await callTool(client, 'browse', {}); - assert.equal( - browse.current.text, - 'MCP capture should feel like the same capture core, not a second product.', - 'Expected browse without an entry id to start from the latest capture.' - ); - assert.equal(browse.older, null, 'Expected no older neighbor after one capture.'); - - const inspect = await callTool(client, 'inspect', { - entryId: capture.entryId, - }); - assert.equal(inspect.entry.entryId, capture.entryId, 'Expected inspect to return the captured entry id.'); - assert.equal( - inspect.entry.text, - 'MCP capture should feel like the same capture core, not a second product.', - 'Expected inspect to preserve the exact raw capture text.' - ); - }); + await withThinkMcpClient( + context, + async ({ client }) => await exerciseCoreMcpRuntime(client) + ); }); test('think MCP capture preserves additive provenance separately from the raw text', async () => { @@ -135,7 +103,48 @@ test('think MCP capture trims additive provenance strings before persistence', a test('think MCP remember, stats, and prompt_metrics expose structured read results', async () => { const context = await createThinkContext(); - const metricsFile = seedPromptMetricsFile(context, [ + const metricsFile = seedMcpPromptMetrics(context); + + runThink(context, ['project notes about warp performance and browse startup']); + runThink(context, ['remembering warp worldlines makes browse startup fast']); + + await withThinkMcpClient( + context, + async ({ client }) => await assertStructuredReadResults(client), + { + THINK_PROMPT_METRICS_FILE: metricsFile, + } + ); +}); + +async function exerciseCoreMcpRuntime(client) { + const text = 'MCP capture should feel like the same capture core, not a second product.'; + const capture = await callTool(client, 'capture', { text }); + + assert.equal(capture.status, 'saved_locally', 'Expected MCP capture to report the normal saved-locally status.'); + assert.equal(capture.backupStatus, 'skipped', 'Expected MCP capture to preserve the normal no-upstream backup behavior.'); + assert.deepEqual( + capture.warnings, + [], + 'Expected completed native v19 capture followthrough to emit no compatibility warning.' + ); + + const recent = await callTool(client, 'recent', {}); + assert.equal(recent.repoPresent, true, 'Expected recent to see the repo bootstrapped by MCP capture.'); + assert.equal(recent.entries.length, 1, 'Expected one recent entry after the first MCP capture.'); + assert.equal(recent.entries[0].text, text, 'Expected recent to surface the exact MCP-captured text.'); + + const browse = await callTool(client, 'browse', {}); + assert.equal(browse.current.text, text, 'Expected browse without an entry id to start from the latest capture.'); + assert.equal(browse.older, null, 'Expected no older neighbor after one capture.'); + + const inspect = await callTool(client, 'inspect', { entryId: capture.entryId }); + assert.equal(inspect.entry.entryId, capture.entryId, 'Expected inspect to return the captured entry id.'); + assert.equal(inspect.entry.text, text, 'Expected inspect to preserve the exact raw capture text.'); +} + +function seedMcpPromptMetrics(context) { + return seedPromptMetricsFile(context, [ makePromptMetric({ sessionId: 'prompt-session-1', ts: '2026-03-29T10:00:00.000Z', @@ -159,43 +168,28 @@ test('think MCP remember, stats, and prompt_metrics expose structured read resul backupState: null, }), ]); +} - runThink(context, ['project notes about warp performance and browse startup']); - runThink(context, ['remembering warp worldlines makes browse startup fast']); +async function assertStructuredReadResults(client) { + const remember = await callTool(client, 'remember', { + brief: true, + limit: 1, + query: 'warp', + }); - await withThinkMcpClient( - context, - async ({ client }) => { - const remember = await callTool(client, 'remember', { - brief: true, - limit: 1, - query: 'warp', - }); - - assert.equal(remember.repoPresent, true, 'Expected remember to read from the local Think repo.'); - assert.equal(remember.matches.length, 1, 'Expected remember --limit semantics to carry into MCP.'); - assert.match( - remember.matches[0].reasonText, - /matched query/, - 'Expected remember to preserve explicit recall receipts.' - ); - - const stats = await callTool(client, 'stats', {}); - assert.equal(stats.repoPresent, true, 'Expected stats to report the repo as present.'); - assert.equal(stats.total, 2, 'Expected stats to count the seeded captures.'); - - const promptMetrics = await callTool(client, 'prompt_metrics', { - bucket: 'day', - }); - assert.equal(promptMetrics.summary.sessions, 2, 'Expected prompt_metrics to report total sessions.'); - assert.equal(promptMetrics.timings.length, 4, 'Expected prompt_metrics to expose the known timing rows.'); - assert.equal(promptMetrics.buckets.length, 1, 'Expected bucketed prompt metrics to return one day bucket.'); - }, - { - THINK_PROMPT_METRICS_FILE: metricsFile, - } - ); -}); + assert.equal(remember.repoPresent, true, 'Expected remember to read from the local Think repo.'); + assert.equal(remember.matches.length, 1, 'Expected remember --limit semantics to carry into MCP.'); + assert.match(remember.matches[0].reasonText, /matched query/, 'Expected explicit recall receipts.'); + + const stats = await callTool(client, 'stats', {}); + assert.equal(stats.repoPresent, true, 'Expected stats to report the repo as present.'); + assert.equal(stats.total, 2, 'Expected stats to count the seeded captures.'); + + const promptMetrics = await callTool(client, 'prompt_metrics', { bucket: 'day' }); + assert.equal(promptMetrics.summary.sessions, 2, 'Expected total prompt sessions.'); + assert.equal(promptMetrics.timings.length, 4, 'Expected the known timing rows.'); + assert.equal(promptMetrics.buckets.length, 1, 'Expected one day bucket.'); +} test('think MCP doctor tool returns structured health checks', async () => { const context = await createThinkContext(); diff --git a/test/acceptance/repair-v17-mind.test.js b/test/acceptance/repair-v17-mind.test.js deleted file mode 100644 index d517b5b8..00000000 --- a/test/acceptance/repair-v17-mind.test.js +++ /dev/null @@ -1,204 +0,0 @@ -import assert from 'node:assert/strict'; -import { spawnSync } from 'node:child_process'; -import { createHash } from 'node:crypto'; -import { existsSync } from 'node:fs'; -import { mkdir, readFile } from 'node:fs/promises'; -import path from 'node:path'; -import test from 'node:test'; - -import { - resolveGitWarpPackageRoot, -} from '../../scripts/repair-v17-mind.mjs'; -import { baseEnv, formatResult, repoRoot } from '../fixtures/runtime.js'; -import { createTempDir } from '../fixtures/tmp.js'; -import { assertSuccess } from '../support/assertions.js'; - -const FIXTURE_METADATA_PATH = path.join(repoRoot, 'test', 'fixtures', 'cas', 'gemini-pre-v17-mind.json'); -const GIT_CAS_ENTRYPOINT = path.join(repoRoot, 'node_modules', '@git-stunts', 'git-cas', 'bin', 'git-cas.js'); -const REPAIR_ENTRYPOINT = path.join(repoRoot, 'scripts', 'repair-v17-mind.mjs'); -const RESTORE_TIMEOUT_MS = 120_000; -const GRAPH = 'think'; - -test('git-cas fixture restores an archived pre-v17 Gemini mind tarball', async () => { - const fixture = await readGeminiFixtureMetadata(); - const { mindDir, tarballPath } = await restoreGeminiFixture(fixture); - - assert.ok(existsSync(tarballPath), 'Expected git-cas restore to recreate the fixture tarball.'); - assert.ok(existsSync(path.join(mindDir, '.git')), 'Expected tar extraction to recreate a Git repository.'); - - const checkpoint = runGit(mindDir, ['rev-parse', '--verify', `refs/warp/${GRAPH}/checkpoints/head`]); - assertSuccess(checkpoint, 'Expected restored fixture to expose the old git-warp checkpoint ref.'); - assert.equal(checkpoint.stdout.trim(), fixture.source.checkpoint); - - const checkpointLog = runGit(mindDir, ['log', '-1', '--format=%B', fixture.source.checkpoint]); - assertSuccess(checkpointLog, 'Expected restored checkpoint to be readable as a Git commit.'); - assert.match( - checkpointLog.stdout, - new RegExp(`^eg-schema: ${fixture.source.schema}$`, 'm'), - 'Expected the restored fixture checkpoint to preserve schema 4.' - ); -}); - -test('repair-v17 mind repairs the restored git-cas fixture when v17 migration is installed', { - timeout: RESTORE_TIMEOUT_MS, -}, async (t) => { - if (!hasV17GitWarpMigration()) { - t.skip('Installed @git-stunts/git-warp package does not expose the v17 checkpoint migration.'); - return; - } - - const fixture = await readGeminiFixtureMetadata(); - const { mindDir } = await restoreGeminiFixture(fixture); - - const before = parseRepairJson( - runRepair(mindDir, ['--dry-run']), - 'Expected dry-run repair to inspect the restored fixture.' - ); - assert.equal(before.ok, true); - assert.equal(before.wouldRepair, true); - assert.equal(before.beforeCheckpoint, fixture.source.checkpoint); - assert.equal(before.upgrade.before, 'needed'); - assert.equal(before.upgrade.rawBefore.previousSchema, fixture.source.schema); - assert.equal(before.upgrade.rawBefore.currentSchema, fixture.expected.upgradedSchema); - - const repaired = parseRepairJson( - runRepair(mindDir), - 'Expected repair to upgrade and materialize the restored fixture.' - ); - assert.equal(repaired.ok, true); - assert.equal(repaired.changed, true); - assert.equal(repaired.beforeCheckpoint, fixture.source.checkpoint); - assert.notEqual(repaired.afterCheckpoint, fixture.source.checkpoint); - assert.match( - repaired.backupRef, - /^refs\/warp\/think\/checkpoints\/pre-v17-upgrade-\d{8}-\d{6}(?:-\d{2})?$/u - ); - assert.equal(repaired.upgrade.before, 'needed'); - assert.equal(repaired.upgrade.after, 'already-current'); - assert.equal(repaired.check.patchesSinceCheckpoint, 0); - assert.equal(repaired.doctor.failures, 0); - - const after = parseRepairJson( - runRepair(mindDir, ['--dry-run']), - 'Expected repaired fixture to be current on a follow-up dry run.' - ); - assert.equal(after.ok, true); - assert.equal(after.wouldRepair, false); - assert.equal(after.upgrade.before, 'already-current'); -}); - -async function readGeminiFixtureMetadata() { - const contents = await readFile(FIXTURE_METADATA_PATH, 'utf8'); - return JSON.parse(contents); -} - -async function restoreGeminiFixture(fixture) { - const parentDir = await createTempDir('think-gemini-pre-v17-fixture-'); - const mindDir = path.join(parentDir, 'mind'); - const tarballPath = path.join(parentDir, fixture.tarball.name); - - assert.ok( - existsSync(GIT_CAS_ENTRYPOINT), - `Expected git-cas test dependency to exist at ${GIT_CAS_ENTRYPOINT}` - ); - - const restore = spawnSync(process.execPath, [ - GIT_CAS_ENTRYPOINT, - '--json', - 'restore', - '--oid', - fixture.treeOid, - '--out', - tarballPath, - '--cwd', - repoRoot, - ], { - cwd: repoRoot, - encoding: 'utf8', - env: fixtureEnv(), - }); - assertSuccess(restore, 'Expected git-cas to restore the archived Gemini fixture tarball.'); - await assertRestoredTarball(tarballPath, fixture); - - await mkdir(mindDir, { recursive: true }); - const extract = spawnSync('tar', ['-xzf', tarballPath, '-C', mindDir], { - cwd: repoRoot, - encoding: 'utf8', - env: fixtureEnv(), - }); - assertSuccess(extract, 'Expected tar to extract the git-cas restored Gemini fixture.'); - - return Object.freeze({ - mindDir, - parentDir, - tarballPath, - }); -} - -function runRepair(mindDir, extraArgs = []) { - return spawnSync(process.execPath, [ - REPAIR_ENTRYPOINT, - '--repo', - mindDir, - '--graph', - GRAPH, - '--json', - ...extraArgs, - ], { - cwd: repoRoot, - encoding: 'utf8', - env: fixtureEnv(), - }); -} - -function runGit(repoDir, args) { - return spawnSync('git', ['-C', repoDir, ...args], { - cwd: repoRoot, - encoding: 'utf8', - env: fixtureEnv(), - }); -} - -function parseRepairJson(result, message) { - assertSuccess(result, message); - try { - return JSON.parse(result.stdout); - } catch (error) { - throw new assert.AssertionError({ - message: `${message}\nCould not parse repair JSON: ${String(error)}\n${formatResult(result)}`, - }); - } -} - -async function assertRestoredTarball(tarballPath, fixture) { - const contents = await readFile(tarballPath); - assert.equal(contents.byteLength, fixture.tarball.bytes); - assert.equal( - createHash('sha256').update(contents).digest('hex'), - fixture.tarball.sha256 - ); -} - -function hasV17GitWarpMigration() { - try { - const packageRoot = resolveGitWarpPackageRoot(); - return existsSync(path.join( - packageRoot, - 'dist', - 'scripts', - 'migrations', - 'v17.0.0', - 'checkpoint-schema-upgrade.js' - )); - } catch { - return false; - } -} - -function fixtureEnv() { - return { - ...process.env, - ...baseEnv, - COPYFILE_DISABLE: '1', - }; -} diff --git a/test/benchmarks/browse-bootstrap.test.js b/test/benchmarks/browse-bootstrap.test.js index 93156ddd..57159e77 100644 --- a/test/benchmarks/browse-bootstrap.test.js +++ b/test/benchmarks/browse-bootstrap.test.js @@ -14,7 +14,7 @@ import { createTempDir } from '../fixtures/tmp.js'; const benchmarkEntrypoint = path.join(repoRoot, 'benchmarks', 'browse-bootstrap.js'); -test('prepareBrowseBootstrap returns the latest capture window from graph-native browse edges', async () => { +test('prepareBrowseBootstrap returns the latest bounded native capture window', async () => { const tempDir = await createTempDir('browse-bootstrap-fixture-'); const repoDir = path.join(tempDir, 'repo'); @@ -26,7 +26,7 @@ test('prepareBrowseBootstrap returns the latest capture window from graph-native const bootstrap = await prepareBrowseBootstrap(repoDir); - assert.equal(bootstrap.ok, true, 'Expected graph-native browse bootstrap to succeed on a populated fixture.'); + assert.equal(bootstrap.ok, true, 'Expected native aggregate browse bootstrap to succeed on a populated fixture.'); assert.equal('entries' in bootstrap, false, 'Expected browse bootstrap to return only the first-paint window, not a preloaded chronology corpus.'); assert.equal(bootstrap.current.id, 'entry:1774023930000-bench-0012', 'Expected bootstrap to start from the latest capture anchor.'); @@ -38,7 +38,7 @@ test('prepareBrowseBootstrap returns the latest capture window from graph-native assert.equal(bootstrap.sessionContext?.sessionCount, 4, 'Expected bootstrap to expose the current session size.'); }); -test('loadBrowseChronologyEntries follows graph-native older edges newest-first', async () => { +test('loadBrowseChronologyEntries follows the bounded native capture index newest-first', async () => { const tempDir = await createTempDir('browse-chronology-fixture-'); const repoDir = path.join(tempDir, 'repo'); @@ -57,7 +57,7 @@ test('loadBrowseChronologyEntries follows graph-native older edges newest-first' assert.match(entries.at(-1)?.text ?? '', /Benchmark thought 1\./, 'Expected chronology traversal to preserve the oldest capture text.'); }); -test('browse benchmark emits a deterministic JSON report for a synthetic fixture graph', () => { +test('browse benchmark emits a deterministic JSON report for a synthetic native aggregate fixture', () => { const result = runBrowseBenchmark(['--json', '--thoughts=12', '--sessions=3', '--warmup=0', '--runs=2']); assert.equal(result.status, 0, formatResult(result)); diff --git a/test/benchmarks/capture-latency.test.js b/test/benchmarks/capture-latency.test.js index 09270be1..057c0999 100644 --- a/test/benchmarks/capture-latency.test.js +++ b/test/benchmarks/capture-latency.test.js @@ -8,10 +8,10 @@ import { repoRoot } from '../fixtures/runtime.js'; const benchmarkScript = path.join(repoRoot, 'benchmarks', 'capture-latency.js'); test('capture latency benchmark runs and produces valid JSON output', () => { - const result = spawnSync(process.execPath, [benchmarkScript, '--json', '--runs=3', '--warmup=1'], { + const result = spawnSync(process.execPath, [benchmarkScript, '--json', '--runs=1', '--warmup=0'], { cwd: repoRoot, encoding: 'utf8', - timeout: 30_000, + timeout: 90_000, }); assert.equal(result.status, 0, `Benchmark exited with status ${result.status}:\n${result.stderr}`); @@ -22,9 +22,9 @@ test('capture latency benchmark runs and produces valid JSON output', () => { assert.equal(typeof report.capturedAt, 'string', 'Expected capturedAt to be an ISO timestamp.'); assert.equal(typeof report.commitSha, 'string', 'Expected commitSha to be present.'); - assert.equal(report.measurement.runCount, 3, 'Expected 3 measured runs.'); - assert.equal(report.measurement.warmupCount, 1, 'Expected 1 warmup run.'); - assert.equal(report.measurement.samplesMs.length, 3, 'Expected 3 sample timings.'); + assert.equal(report.measurement.runCount, 1, 'Expected 1 measured run.'); + assert.equal(report.measurement.warmupCount, 0, 'Expected no warmup run.'); + assert.equal(report.measurement.samplesMs.length, 1, 'Expected 1 sample timing.'); assert.ok(report.measurement.samplesMs.every((s) => typeof s === 'number' && s > 0), 'Expected all samples to be positive numbers.'); assert.equal(typeof report.summary.minMs, 'number', 'Expected minMs in summary.'); @@ -39,10 +39,10 @@ test('capture latency benchmark runs and produces valid JSON output', () => { }); test('capture latency benchmark produces readable human output by default', () => { - const result = spawnSync(process.execPath, [benchmarkScript, '--runs=2', '--warmup=0'], { + const result = spawnSync(process.execPath, [benchmarkScript, '--runs=1', '--warmup=0'], { cwd: repoRoot, encoding: 'utf8', - timeout: 30_000, + timeout: 90_000, }); assert.equal(result.status, 0, `Benchmark exited with status ${result.status}:\n${result.stderr}`); @@ -52,10 +52,10 @@ test('capture latency benchmark produces readable human output by default', () = }); test('capture latency benchmark uses an isolated temp repo', () => { - const result = spawnSync(process.execPath, [benchmarkScript, '--json', '--runs=2', '--warmup=0'], { + const result = spawnSync(process.execPath, [benchmarkScript, '--json', '--runs=1', '--warmup=0'], { cwd: repoRoot, encoding: 'utf8', - timeout: 30_000, + timeout: 90_000, }); assert.equal(result.status, 0); diff --git a/test/benchmarks/capture-profile.test.js b/test/benchmarks/capture-profile.test.js index db9c1a06..9a945c41 100644 --- a/test/benchmarks/capture-profile.test.js +++ b/test/benchmarks/capture-profile.test.js @@ -8,10 +8,10 @@ import { repoRoot } from '../fixtures/runtime.js'; const benchmarkScript = path.join(repoRoot, 'benchmarks', 'capture-latency.js'); test('capture latency benchmark with --profile produces phase-level timing breakdown', () => { - const result = spawnSync(process.execPath, [benchmarkScript, '--json', '--profile', '--runs=2', '--warmup=1'], { + const result = spawnSync(process.execPath, [benchmarkScript, '--json', '--profile', '--runs=1', '--warmup=0'], { cwd: repoRoot, encoding: 'utf8', - timeout: 30_000, + timeout: 90_000, }); assert.equal(result.status, 0, `Benchmark exited with status ${result.status}:\n${result.stderr}`); @@ -27,13 +27,21 @@ test('capture latency benchmark with --profile produces phase-level timing break assert.equal(typeof report.profile[phase].medianMs, 'number', `Expected medianMs for ${phase}.`); assert.ok(report.profile[phase].medianMs >= 0, `Expected non-negative medianMs for ${phase}.`); } + assert.ok( + report.profile.repoEnsure.medianMs > 0, + 'Expected profile events to report repository setup work.' + ); + assert.ok( + report.profile.rawSave.medianMs > 0, + 'Expected profile events to report native capture write work.' + ); }); test('capture latency profile phases sum to approximately the total end-to-end time', () => { - const result = spawnSync(process.execPath, [benchmarkScript, '--json', '--profile', '--runs=3', '--warmup=1'], { + const result = spawnSync(process.execPath, [benchmarkScript, '--json', '--profile', '--runs=1', '--warmup=0'], { cwd: repoRoot, encoding: 'utf8', - timeout: 30_000, + timeout: 90_000, }); assert.equal(result.status, 0); @@ -50,10 +58,10 @@ test('capture latency profile phases sum to approximately the total end-to-end t }); test('capture latency --profile human output shows phase breakdown', () => { - const result = spawnSync(process.execPath, [benchmarkScript, '--profile', '--runs=2', '--warmup=0'], { + const result = spawnSync(process.execPath, [benchmarkScript, '--profile', '--runs=1', '--warmup=0'], { cwd: repoRoot, encoding: 'utf8', - timeout: 30_000, + timeout: 90_000, }); assert.equal(result.status, 0, `Benchmark exited with status ${result.status}:\n${result.stderr}`); diff --git a/test/fixtures/think.js b/test/fixtures/think.js index 9a3659b7..3b5c822a 100644 --- a/test/fixtures/think.js +++ b/test/fixtures/think.js @@ -8,7 +8,7 @@ import { createTempDir } from './tmp.js'; export async function createThinkContext({ upstream = 'none' } = {}) { const homeDir = await createTempDir('think-home-'); const thinkDir = path.join(homeDir, '.think'); - const localRepoDir = path.join(thinkDir, 'repo'); + const localRepoDir = path.join(thinkDir, 'james'); let upstreamUrl = ''; if (upstream === 'bare') { diff --git a/test/ports/browse-window.test.js b/test/ports/browse-window.test.js index eed98b11..f08ec798 100644 --- a/test/ports/browse-window.test.js +++ b/test/ports/browse-window.test.js @@ -1,12 +1,8 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { stringifyJson } from '../../src/json.js'; -import { CAPTURE_READ_MODEL_ID } from '../../src/store/constants.js'; import { getBrowseWindowForRead } from '../../src/store.js'; -const ENCODER = new TextEncoder(); - test('browse window hydrates only the selected capture and adjacent chronology entries', async () => { const read = createCaptureRead([ capture('entry:1774434000000-far-newer', 'far newer', '2026-03-25T10:20:00.000Z'), @@ -22,60 +18,40 @@ test('browse window hydrates only the selected capture and adjacent chronology e assert.equal(window.newer.text, 'newer neighbor'); assert.equal(window.older.text, 'older neighbor'); assert.deepEqual( - [...read.contentReads].sort(), + [...read.documentReads].sort(), [ 'entry:1774432800000-older', 'entry:1774432860000-current', 'entry:1774432920000-newer', ], - 'Expected browse to avoid hydrating unrelated capture bodies.' + 'Expected browse to avoid exact reads for unrelated capture documents.' ); }); function capture(id, text, createdAt) { - return { + return Object.freeze({ id, + kind: 'capture', text, - props: { - kind: 'capture', - writerId: 'test', - createdAt, - sortKey: id.slice('entry:'.length), - }, - }; + writerId: 'test', + createdAt, + sortKey: id.slice('entry:'.length), + }); } function createCaptureRead(entries) { - const propsById = new Map(entries.map((entry) => [entry.id, entry.props])); - const textById = new Map(entries.map((entry) => [entry.id, entry.text])); - const contentReads = new Set(); - + const documentsById = new Map(entries.map(entry => [entry.id, entry])); + const documentReads = new Set(); return { - contentReads, + documentReads, repoDir: '/tmp/think-fake-read', - view: { - getNodeProps(nodeId) { - if (nodeId === CAPTURE_READ_MODEL_ID) { - return { - kind: 'capture_read_model', - latestCaptureId: entries[0]?.id ?? null, - totalCaptures: entries.length, - recentCaptureRefsJson: stringifyJson(entries.map((entry) => ({ - id: entry.id, - createdAt: entry.props.createdAt, - sortKey: entry.props.sortKey, - }))), - }; - } - return propsById.get(nodeId) ?? null; - }, - query() { - throw new Error('Expected browse window to use bounded read-model refs, not graph queries.'); - }, + listDocumentsByKind(kind, { limit }) { + assert.equal(kind, 'capture'); + return entries.slice(0, limit); }, - readContent(nodeId) { - contentReads.add(nodeId); - return ENCODER.encode(textById.get(nodeId) ?? ''); + readDocument(nodeId) { + documentReads.add(nodeId); + return documentsById.get(nodeId) ?? null; }, }; } diff --git a/test/ports/capture-context.test.js b/test/ports/capture-context.test.js index e38b966f..9139585b 100644 --- a/test/ports/capture-context.test.js +++ b/test/ports/capture-context.test.js @@ -1,13 +1,15 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { GitGraphAdapter, openWarpWorldline } from '@git-stunts/git-warp'; +import { Runtime } from '@git-stunts/git-warp'; -import { createThinkPlumbing, ensureGitRepo } from '../../src/git.js'; -import { getCaptureAmbientContext, getAmbientProjectContext } from '../../src/project-context.js'; +import { ensureGitRepo } from '../../src/git.js'; +import { getAmbientProjectContext } from '../../src/project-context.js'; import { finalizeCapturedThought, + getGraphModelStatus, GRAPH_NAME, inspectRawEntry, + listRecent, openProductReadHandle, saveAnnotation, saveRawCapture, @@ -15,11 +17,16 @@ import { startReflect, } from '../../src/store.js'; import { createWriterId } from '../../src/store/model.js'; +import { thinkMemory } from '../../src/generated/think-memory.generated.js'; +import { + getStoredEntry, + listEntriesByKind, +} from '../../src/store/runtime.js'; import { createGitRepo, runGit } from '../fixtures/git.js'; import { createTempDir } from '../fixtures/tmp.js'; import { formatResult } from '../fixtures/runtime.js'; -test('saveRawCapture writes cwd receipts first and defers git enrichment to followthrough', async () => { +test('saveRawCapture stores complete ambient receipts in the native capture document', async () => { const localRepoDir = await createTempDir('think-capture-context-'); await ensureGitRepo(localRepoDir); @@ -46,37 +53,57 @@ test('saveRawCapture writes cwd receipts first and defers git enrichment to foll ); const entry = await saveRawCapture(localRepoDir, 'capture should stay cheap', { - ambientContext: getCaptureAmbientContext(projectRepoDir), + ambientContext: getAmbientProjectContext(projectRepoDir), }); const readBeforeFollowthrough = await openProductReadHandle(localRepoDir); - const savedBeforeFollowthrough = await readBeforeFollowthrough.view.getNodeProps(entry.id); + const savedBeforeFollowthrough = await getStoredEntry(readBeforeFollowthrough, entry.id); assert.ok(savedBeforeFollowthrough, 'Expected saved raw entry to be readable immediately after local save.'); - assert.equal(savedBeforeFollowthrough.ambientCwd, projectRepoDir, 'Expected the cheap capture path to still record cwd immediately.'); - assert.equal(savedBeforeFollowthrough.ambientGitRoot ?? null, null, 'Expected git root enrichment to be deferred until followthrough.'); - assert.equal(savedBeforeFollowthrough.ambientGitRemote ?? null, null, 'Expected git remote enrichment to be deferred until followthrough.'); - assert.equal(savedBeforeFollowthrough.ambientGitBranch ?? null, null, 'Expected git branch enrichment to be deferred until followthrough.'); + assert.equal(savedBeforeFollowthrough.ambientCwd, projectRepoDir); + assert.equal(savedBeforeFollowthrough.ambientGitRoot, String(gitRoot.stdout || '').trim()); + assert.equal(savedBeforeFollowthrough.ambientGitRemote, remoteUrl); + assert.equal(savedBeforeFollowthrough.ambientGitBranch, String(branch.stdout || '').trim()); const followthrough = await finalizeCapturedThought(localRepoDir, entry.id, { ambientContext: getAmbientProjectContext(projectRepoDir), }); const readAfterFollowthrough = await openProductReadHandle(localRepoDir); - const savedAfterFollowthrough = await readAfterFollowthrough.view.getNodeProps(entry.id); + const savedAfterFollowthrough = await getStoredEntry(readAfterFollowthrough, entry.id); assert.ok(followthrough.entry, 'Expected followthrough to keep the saved capture entry available.'); assert.ok(savedAfterFollowthrough, 'Expected saved raw entry to remain readable after followthrough.'); - assert.equal(savedAfterFollowthrough.ambientCwd, projectRepoDir, 'Expected followthrough to preserve the original cwd receipt.'); - assert.equal( - savedAfterFollowthrough.ambientGitRoot, - String(gitRoot.stdout || '').trim(), - 'Expected followthrough to backfill the git root receipt.' - ); - assert.equal(savedAfterFollowthrough.ambientGitRemote, remoteUrl, 'Expected followthrough to backfill the git remote receipt.'); - assert.equal( - savedAfterFollowthrough.ambientGitBranch, - String(branch.stdout || '').trim(), - 'Expected followthrough to backfill the current git branch receipt.' + assert.equal(savedAfterFollowthrough.ambientCwd, projectRepoDir); + assert.equal(savedAfterFollowthrough.ambientGitRoot, String(gitRoot.stdout || '').trim()); + assert.equal(savedAfterFollowthrough.ambientGitRemote, remoteUrl); + assert.equal(savedAfterFollowthrough.ambientGitBranch, String(branch.stdout || '').trim()); +}); + +test('new-mind raw capture initializes History metadata before followthrough', async () => { + const localRepoDir = await createTempDir('think-capture-bootstrap-'); + await ensureGitRepo(localRepoDir); + + await saveRawCapture(localRepoDir, 'bootstrap History metadata atomically', { + initializeGraphModel: true, + }); + + const status = await getGraphModelStatus(localRepoDir); + assert.equal(status.migrationRequired, false); + assert.equal(status.currentGraphModelVersion, status.requiredGraphModelVersion); +}); + +test('raw captures preserve the bounded recent index before followthrough', async () => { + const localRepoDir = await createTempDir('think-capture-index-'); + await ensureGitRepo(localRepoDir); + + await saveRawCapture(localRepoDir, 'first pending followthrough'); + await saveRawCapture(localRepoDir, 'second pending followthrough'); + + const recent = await listRecent(localRepoDir, { count: 2 }); + assert.deepEqual( + recent.entries.map((entry) => entry.text), + ['second pending followthrough', 'first pending followthrough'] ); + assert.equal(recent.total, 2); }); test('saveRawCapture retries after the cached writer ref is advanced externally', async () => { @@ -84,16 +111,11 @@ test('saveRawCapture retries after the cached writer ref is advanced externally' await ensureGitRepo(localRepoDir); await saveRawCapture(localRepoDir, 'seed capture before external writer advance'); - const externalWorldline = await openExternalWorldline(localRepoDir); - await externalWorldline.commit((patch) => { - patch - .addNode('external:writer-advance') - .setProperty('external:writer-advance', 'kind', 'external_fixture'); - }); + await advanceWriterExternally(localRepoDir, 'external:writer-advance'); const entry = await saveRawCapture(localRepoDir, 'capture should retry after writer ref conflict'); const read = await openProductReadHandle(localRepoDir); - const saved = await read.view.getNodeProps(entry.id); + const saved = await getStoredEntry(read, entry.id); assert.ok(saved, 'Expected retrying raw capture to be committed after the writer ref advanced.'); assert.equal(saved.kind, 'capture', 'Expected retried write to preserve capture semantics.'); @@ -104,12 +126,7 @@ test('saveAnnotation retries after the cached writer ref is advanced externally' await ensureGitRepo(localRepoDir); const entry = await saveRawCapture(localRepoDir, 'annotation retry seed capture'); - const externalWorldline = await openExternalWorldline(localRepoDir); - await externalWorldline.commit((patch) => { - patch - .addNode('external:annotation-writer-advance') - .setProperty('external:annotation-writer-advance', 'kind', 'external_fixture'); - }); + await advanceWriterExternally(localRepoDir, 'external:annotation-writer-advance'); const result = await saveAnnotation(localRepoDir, entry.id, 'annotation should retry after writer ref conflict'); const inspected = await inspectRawEntry(localRepoDir, entry.id); @@ -129,22 +146,12 @@ test('reflect writes retry after the cached writer ref is advanced externally', localRepoDir, 'We should redesign browse startup because transitional reads can hide latency.' ); - const firstExternalWorldline = await openExternalWorldline(localRepoDir); - await firstExternalWorldline.commit((patch) => { - patch - .addNode('external:reflect-start-writer-advance') - .setProperty('external:reflect-start-writer-advance', 'kind', 'external_fixture'); - }); + await advanceWriterExternally(localRepoDir, 'external:reflect-start-writer-advance'); const started = await startReflect(localRepoDir, entry.id, { promptType: 'challenge' }); assert.equal(started.ok, true, 'Expected reflect start to retry and create a session after writer ref conflict.'); - const secondExternalWorldline = await openExternalWorldline(localRepoDir); - await secondExternalWorldline.commit((patch) => { - patch - .addNode('external:reflect-reply-writer-advance') - .setProperty('external:reflect-reply-writer-advance', 'kind', 'external_fixture'); - }); + await advanceWriterExternally(localRepoDir, 'external:reflect-reply-writer-advance'); const saved = await saveReflectResponse( localRepoDir, @@ -154,14 +161,26 @@ test('reflect writes retry after the cached writer ref is advanced externally', assert.ok(saved, 'Expected reflect reply to retry and save after writer ref conflict.'); assert.equal(saved.sessionId, started.sessionId, 'Expected retried reflect reply to preserve session lineage.'); + + const read = await openProductReadHandle(localRepoDir); + const sessions = await listEntriesByKind(read, 'reflect_session'); + assert.equal(sessions.length, 1, 'Expected the native reflect-session index to retain one session.'); + assert.equal( + sessions[0].stepCount, + 1, + 'Expected the indexed reflect session to advance with the exact session document.' + ); }); -async function openExternalWorldline(repoDir) { - return await openWarpWorldline({ - persistence: new GitGraphAdapter({ - plumbing: createThinkPlumbing(repoDir), - }), - worldlineName: GRAPH_NAME, - writerId: createWriterId(), +async function advanceWriterExternally(repoDir, nodeId) { + const runtime = await Runtime.open({ + at: repoDir, + writer: createWriterId(), }); + try { + const lane = await runtime.lane(GRAPH_NAME); + await lane.write(thinkMemory.intents.declareMemoryObject({ subject: nodeId })); + } finally { + await runtime.close(); + } } diff --git a/test/ports/convert-v19-mind.test.js b/test/ports/convert-v19-mind.test.js new file mode 100644 index 00000000..78a8764e --- /dev/null +++ b/test/ports/convert-v19-mind.test.js @@ -0,0 +1,315 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { promisify } from 'node:util'; + +import { + buildNativeInventory, + convertV19Mind, + createInventorySnapshot, + mergeSourceInventories, + parseConvertArgs, + parseInventorySnapshot, + projectNativeInventory, +} from '../../scripts/convert-v19-mind.mjs'; + +const execFileAsync = promisify(execFile); + +test('native v19 converter rejects positional repository targets', () => { + assert.throws( + () => parseConvertArgs(['/tmp/mind', '--dry-run']), + /Unknown argument/ + ); +}); + +test('native v19 converter parses explicit repository options', () => { + assert.deepEqual( + parseConvertArgs([ + '--source', + '/tmp/legacy-mind', + '--inventory-out', + '/tmp/legacy-mind.inventory.json', + '--target', + '/tmp/native-mind', + '--dry-run', + '--json', + ]), + { + dryRun: true, + help: false, + inventoryIn: null, + inventoryOut: '/tmp/legacy-mind.inventory.json', + json: true, + source: '/tmp/legacy-mind', + target: '/tmp/native-mind', + } + ); +}); + +test('legacy evacuation materializes once without observer property scans', async () => { + const source = await readFile( + path.resolve('scripts/convert-v19-mind.mjs'), + 'utf8' + ); + + assert.doesNotMatch(source, /\bcreateManyObserver\b/u); + assert.doesNotMatch(source, /\breading\.property\b/u); + assert.doesNotMatch(source, /\.observe\(/u); + assert.match(source, /V18CheckpointMigrationCodec/u); + assert.match(source, /return await graph\.materialize\(\)/u); + assert.match(source, /Reflect\.deleteProperty\(withoutTrie, 'trie'\)/u); + assert.match(source, /\[\.\.\.state\.nodeAlive\.elements\(\)\]\.sort\(\)/u); +}); + +test('native inventory preserves domain documents and native outgoing edges', () => { + const inventory = buildNativeInventory(new Map([ + ['entry:one', legacyRecord({ + props: { + kind: 'capture', + createdAt: '2026-07-28T00:00:00.000Z', + sortKey: '001', + }, + text: 'retained thought', + outgoing: [{ id: 'thought:one', label: 'expresses' }], + })], + ['thought:one', legacyRecord({ + props: { kind: 'thought' }, + })], + ])); + + assert.deepEqual(inventory.documents[0], { + id: 'entry:one', + kind: 'capture', + createdAt: '2026-07-28T00:00:00.000Z', + sortKey: '001', + text: 'retained thought', + }); + assert.deepEqual(inventory.edges, [ + { from: 'entry:one', to: 'thought:one', label: 'expresses' }, + ]); + assert.equal(inventory.byKind.get('capture').length, 1); + assert.equal(inventory.byKind.get('thought').length, 1); +}); + +test('native inventory snapshot round-trips with a stable manifest checksum', () => { + const inventory = buildNativeInventory(new Map([ + ['entry:one', legacyRecord({ + props: { + kind: 'capture', + createdAt: '2026-07-28T00:00:00.000Z', + sortKey: '001', + }, + text: 'retained thought', + outgoing: [{ id: 'thought:one', label: 'expresses' }], + })], + ['thought:one', legacyRecord({ + props: { kind: 'thought' }, + })], + ])); + const snapshot = createInventorySnapshot({ + inventory, + sourceDir: '/tmp/legacy-mind', + sourceRefsSha256: 'a'.repeat(64), + }); + const parsed = parseInventorySnapshot(JSON.stringify(snapshot)); + + assert.equal(parsed.snapshot.manifestSha256, snapshot.manifestSha256); + assert.deepEqual(parsed.snapshot.summary, { + documentCount: 2, + edgeCount: 1, + kinds: { + capture: 1, + thought: 1, + }, + }); + assert.deepEqual(parsed.inventory.documents, inventory.documents); + assert.deepEqual(parsed.inventory.edges, inventory.edges); +}); + +test('native inventory snapshot rejects application-data tampering', () => { + const inventory = buildNativeInventory(new Map([ + ['entry:one', legacyRecord({ + props: { kind: 'capture' }, + text: 'retained thought', + })], + ])); + const snapshot = createInventorySnapshot({ + inventory, + sourceDir: '/tmp/legacy-mind', + sourceRefsSha256: 'b'.repeat(64), + }); + const tampered = structuredClone(snapshot); + tampered.documents[0].text = 'changed after inventory'; + + assert.throws( + () => parseInventorySnapshot(JSON.stringify(tampered)), + /checksum does not match/ + ); +}); + +test('native projection retains user records and drops recomputable graph state', () => { + const projected = projectNativeInventory(projectionSourceInventory()); + + assert.deepEqual( + projected.documents.map(document => document.id), + ['annotation:one', 'entry:one', 'thought:orphan'] + ); + assert.deepEqual(projected.edges, []); + assert.equal(projected.byKind.get('capture').length, 1); + assert.equal(projected.byKind.get('annotation').length, 1); + assert.equal(projected.byKind.get('thought').length, 1); +}); + +test('mixed source inventory preserves exact native documents after legacy data', () => { + const legacy = buildNativeInventory(new Map([ + ['entry:legacy', legacyRecord({ + props: { kind: 'capture', sortKey: '001' }, + text: 'legacy capture', + })], + ])); + const native = Object.freeze({ + id: 'entry:native', + kind: 'capture', + sortKey: '002', + text: 'native capture', + }); + + const merged = mergeSourceInventories(legacy, [native]); + + assert.deepEqual( + merged.byKind.get('capture').map(document => document.id), + ['entry:legacy', 'entry:native'] + ); + assert.deepEqual(merged.documents, [legacy.documents[0], native]); +}); + +test('native inventory imports into and verifies an empty real Git repository', { + timeout: 60_000, +}, async (context) => { + const { inventoryPath, targetDir } = await createConversionFixture(context); + const report = await convertV19Mind({ + dryRun: false, + inventoryIn: inventoryPath, + inventoryOut: null, + source: null, + target: targetDir, + }); + + assert.equal(report.status, 'converted'); + assert.equal(report.verified, true); + assert.equal(report.documentCount, 2); + assert.equal(report.edgeCount, 0); + assert.equal(report.verification.captureCount, 1); + assert.ok(report.verification.samples.every(sample => sample.matched)); +}); + +function projectionSourceInventory() { + return buildNativeInventory(new Map([ + projectionCaptureRecord(), + ['thought:derived-one', legacyRecord({ + props: { kind: 'thought' }, + text: 'duplicate thought projection', + })], + ['thought:orphan', legacyRecord({ + props: { kind: 'thought' }, + text: 'unmatched historical thought', + })], + projectionAnnotationRecord(), + ['artifact:auto-tags', legacyRecord({ + props: { kind: 'auto_tags' }, + outgoing: [{ id: 'thought:derived-one', label: 'derived_from' }], + })], + ['read_model:capture:index', legacyRecord({ + props: { kind: 'capture_read_model' }, + })], + ['classification:idea', legacyRecord({ + props: { kind: 'classification', name: 'idea' }, + })], + ['meta:graph', legacyRecord({ + props: { kind: 'graph_meta', graphModelVersion: 4 }, + })], + ])); +} + +function projectionCaptureRecord() { + return ['entry:one', legacyRecord({ + props: { + kind: 'capture', + thoughtId: 'thought:derived-one', + }, + text: 'canonical user capture', + outgoing: [ + { id: 'thought:derived-one', label: 'expresses' }, + { id: 'annotation:one', label: 'has_annotation' }, + { id: 'annotation:one', label: 'older' }, + ], + })]; +} + +function projectionAnnotationRecord() { + return ['annotation:one', legacyRecord({ + props: { + kind: 'annotation', + targetEntryId: 'entry:one', + }, + text: 'human annotation', + outgoing: [{ id: 'entry:one', label: 'annotates' }], + })]; +} + +async function createConversionFixture(context) { + const root = await mkdtemp(path.join(os.tmpdir(), 'think-native-converter-')); + context.after(async () => { + await rm(root, { recursive: true, force: true }); + }); + const targetDir = path.join(root, 'native-mind'); + const inventoryPath = path.join(root, 'mind.inventory.json'); + await execFileAsync('git', ['init', targetDir]); + await execFileAsync('git', ['-C', targetDir, 'config', 'user.name', 'think']); + await execFileAsync( + 'git', + ['-C', targetDir, 'config', 'user.email', 'think@local.invalid'] + ); + await writeConversionInventory(inventoryPath); + return { inventoryPath, targetDir }; +} + +async function writeConversionInventory(inventoryPath) { + const inventory = buildNativeInventory(new Map([ + ['entry:one', legacyRecord({ + props: { + kind: 'capture', + createdAt: '2026-07-28T00:00:00.000Z', + sortKey: '001', + }, + text: 'retained thought', + outgoing: [{ id: 'thought:one', label: 'expresses' }], + })], + ['thought:one', legacyRecord({ + props: { kind: 'thought' }, + })], + ])); + const snapshot = createInventorySnapshot({ + inventory, + sourceDir: '/disposable/legacy-mind', + sourceRefsSha256: 'c'.repeat(64), + }); + await writeFile(inventoryPath, JSON.stringify(snapshot), 'utf8'); +} + +function legacyRecord({ + props, + text = null, + incoming = [], + outgoing = [], +}) { + return Object.freeze({ + props: Object.freeze(props), + text, + incoming: Object.freeze(incoming), + outgoing: Object.freeze(outgoing), + }); +} diff --git a/test/ports/enrichment-cache.test.js b/test/ports/enrichment-cache.test.js index 537a553b..ad883834 100644 --- a/test/ports/enrichment-cache.test.js +++ b/test/ports/enrichment-cache.test.js @@ -11,6 +11,11 @@ import { loadSearchIndex, } from '../../src/store/queries.js'; import { runEnrichmentPipeline } from '../../src/store/enrichment/runner.js'; +import { + getSingleNeighborId, + listEntriesByKind, + openProductReadHandle, +} from '../../src/store/runtime.js'; import { createTempDir } from '../fixtures/tmp.js'; test('enrichment invalidates the per-repo search index after creating keyword nodes', async () => { @@ -37,6 +42,9 @@ test('enrichment invalidates the per-repo search index after creating keyword no 'Expected enrichment to count both auto_tags and semantic_parse receipts.' ); + await assertNativeCaptureRootedEnrichment(repoDir, entry); + await assertRepeatedEnrichmentIsNoop(repoDir); + const after = await loadSearchIndex(repoDir); assert.ok( after.search('performance').includes('performance'), @@ -88,3 +96,47 @@ async function createEnrichedRepo(prefix, thought) { return repoDir; } + +async function assertRepeatedEnrichmentIsNoop(repoDir) { + assert.deepEqual( + await runEnrichmentPipeline(repoDir), + { + capturesProcessed: 0, + topicNodesCreated: 0, + keywordNodesCreated: 0, + aboutEdgesAdded: 0, + mentionsEdgesAdded: 0, + classifiedEdgesAdded: 0, + receiptsCreated: 0, + promotedTopics: [], + }, + 'Expected the persisted native capture cursor to make a repeated enrichment run a no-op.' + ); + const tagReceipts = await listEntriesByKind( + await openProductReadHandle(repoDir), + 'auto_tags' + ); + assert.equal( + tagReceipts.length, + 1, + 'Expected repeated enrichment not to duplicate its native receipt.' + ); +} + +async function assertNativeCaptureRootedEnrichment(repoDir, entry) { + const read = await openProductReadHandle(repoDir); + const tagReceipts = await listEntriesByKind(read, 'auto_tags'); + assert.equal(tagReceipts.length, 1, 'Expected one native auto-tags receipt for the capture.'); + assert.equal(tagReceipts[0].primaryInputKind, 'capture'); + assert.equal(tagReceipts[0].primaryInputId, entry.id); + assert.equal( + await getSingleNeighborId(read, tagReceipts[0].id, 'outgoing', 'derived_from'), + entry.id, + 'Expected the native receipt to derive from the stored capture object.' + ); + assert.equal( + await read.memory.exists(entry.thoughtId), + false, + 'Expected content identity to remain a capture fact rather than a legacy thought node.' + ); +} diff --git a/test/ports/mcp-service.test.js b/test/ports/mcp-service.test.js index 640f9d45..b4e51bfd 100644 --- a/test/ports/mcp-service.test.js +++ b/test/ports/mcp-service.test.js @@ -19,7 +19,7 @@ test('MCP capture returns saved locally when post-save followthrough defers', as 'hasGitRepo', 'ensureGitRepo', 'getCwd', - 'getCaptureAmbientContext', + 'getAmbientProjectContext', 'saveRawCapture', 'getGraphModelStatus', 'waitForFollowthrough', @@ -61,8 +61,7 @@ function createDeferredFollowthroughDeps(calls) { return { ensureGitRepo: (repoDir) => record(calls, 'ensureGitRepo', repoDir), finalizeCapturedThought: () => record(calls, 'finalizeCapturedThought'), - getAmbientProjectContext: () => record(calls, 'getAmbientProjectContext'), - getCaptureAmbientContext: (cwd) => ({ cwd: record(calls, 'getCaptureAmbientContext', cwd) }), + getAmbientProjectContext: (cwd) => ({ cwd: record(calls, 'getAmbientProjectContext', cwd) }), getCwd: () => record(calls, 'getCwd', '/tmp/project'), getGraphModelStatus: () => { record(calls, 'getGraphModelStatus'); diff --git a/test/ports/minds.test.js b/test/ports/minds.test.js index 06ba33f5..3718e142 100644 --- a/test/ports/minds.test.js +++ b/test/ports/minds.test.js @@ -52,24 +52,29 @@ test('discoverMinds ignores directories without git repos', async () => { assert.equal(minds[0].name, 'valid'); }); -test('discoverMinds labels ~/.think/repo as "default"', async () => { +test('discoverMinds labels ~/.think/james as "default" and keeps repo discoverable', async () => { const thinkDir = await createTempDir('think-minds-'); - const repoDir = path.join(thinkDir, 'repo'); - mkdirSync(repoDir, { recursive: true }); - execSync('git init', { cwd: repoDir, stdio: 'ignore' }); + for (const name of ['james', 'repo']) { + const mindDir = path.join(thinkDir, name); + mkdirSync(mindDir, { recursive: true }); + execSync('git init', { cwd: mindDir, stdio: 'ignore' }); + } const minds = discoverMinds(thinkDir); - assert.equal(minds.length, 1); - assert.equal(minds[0].name, 'default', 'Expected "repo" directory to display as "default".'); + assert.equal(minds.length, 2); + assert.equal(minds[0].name, 'default', 'Expected "james" directory to display as "default".'); + assert.equal(minds[0].repoDir, path.join(thinkDir, 'james')); assert.equal(minds[0].isDefault, true); + assert.equal(minds[1].name, 'repo', 'Expected the retained "repo" directory to remain discoverable.'); + assert.equal(minds[1].isDefault, false); }); test('discoverMinds sorts with default first, then alphabetical', async () => { const thinkDir = await createTempDir('think-minds-'); - for (const name of ['work', 'repo', 'claude']) { + for (const name of ['work', 'james', 'claude']) { const mindDir = path.join(thinkDir, name); mkdirSync(mindDir, { recursive: true }); execSync('git init', { cwd: mindDir, stdio: 'ignore' }); diff --git a/test/ports/native-index.test.js b/test/ports/native-index.test.js new file mode 100644 index 00000000..659764a8 --- /dev/null +++ b/test/ports/native-index.test.js @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + listIndexedMemoryDocuments, + NATIVE_INDEX_PAGE_SIZE, +} from '../../src/store/native-index.js'; +import { encodeNativeDocument } from '../../src/store/native-document.js'; + +test('recent native index pages read concurrently and preserve newest-first order', async () => { + const fixture = createIndexFixture({ total: (NATIVE_INDEX_PAGE_SIZE * 2) + 1 }); + + const documents = await listIndexedMemoryDocuments( + '/tmp/think-native-index', + 'capture', + { + limit: NATIVE_INDEX_PAGE_SIZE + 2, + memory: fixture.memory, + } + ); + + assert.equal(fixture.startedBeforeRelease, 3); + assert.deepEqual(fixture.pageReads, [ + 'read_model:v19:index:capture:page:00000002', + 'read_model:v19:index:capture:page:00000001', + 'read_model:v19:index:capture:page:00000000', + ]); + assert.equal(documents.length, NATIVE_INDEX_PAGE_SIZE + 2); + assert.equal(documents[0].id, 'entry:128'); + assert.equal(documents.at(-1).id, 'entry:63'); +}); + +function createIndexFixture({ total }) { + const pageReads = []; + const fixture = { + pageReads, + startedBeforeRelease: 0, + }; + const reader = createDelayedPageReader({ fixture, pageReads, total }); + fixture.memory = { + hasHistory: () => true, + captureBoundedReader: () => reader, + }; + return fixture; +} + +function createDelayedPageReader({ fixture, pageReads, total }) { + const pages = indexPages(); + let release; + const released = new Promise(resolve => { + release = resolve; + }); + const releaseTimer = setTimeout(() => { + fixture.startedBeforeRelease = pageReads.length; + release(); + }, 10); + releaseTimer.unref(); + + return { + memoryIndex() { + return encodeNativeDocument('memory-index', { + headPage: 2, + latestId: `entry:${total - 1}`, + total, + }); + }, + async memoryIndexPage(id) { + pageReads.push(id); + await released; + const pageNumber = Number.parseInt(id.slice(-8), 10); + return encodeNativeDocument('memory-index-page', pages.get(pageNumber)); + }, + }; +} + +function indexPages() { + return new Map([ + [0, page(0, NATIVE_INDEX_PAGE_SIZE)], + [1, page(NATIVE_INDEX_PAGE_SIZE, NATIVE_INDEX_PAGE_SIZE)], + [2, page(NATIVE_INDEX_PAGE_SIZE * 2, 1)], + ]); +} + +function page(start, count) { + return Object.freeze({ + entries: Object.freeze( + Array.from({ length: count }, (_, offset) => Object.freeze({ + id: `entry:${start + offset}`, + kind: 'capture', + sortKey: String(start + offset).padStart(4, '0'), + text: `capture ${start + offset}`, + })) + ), + }); +} diff --git a/test/ports/native-v19-sdk.test.js b/test/ports/native-v19-sdk.test.js new file mode 100644 index 00000000..dcccd915 --- /dev/null +++ b/test/ports/native-v19-sdk.test.js @@ -0,0 +1,91 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { thinkMemory } from '../../src/generated/think-memory.generated.js'; +import { + decodeNativeDocument, + encodeNativeDocument, + INDEX_DOCUMENT_KEY, + INDEX_PAGE_DOCUMENT_KEY, +} from '../../src/store/native-document.js'; +import { NativeMemoryCoordinateReader } from '../../src/store/native-runtime.js'; + +test('generated Think SDK emits native v19 Intent values', () => { + const declaration = thinkMemory.intents.declareMemoryObject({ + subject: 'entry:test', + }); + const document = thinkMemory.intents.storeMemoryDocument({ + subject: 'entry:test', + value: new Uint8Array([1, 2, 3]), + }); + assert.deepEqual(declaration.descriptor, { + kind: 'node.add', + subject: 'entry:test', + }); + assert.equal(document.descriptor.kind, 'property.set'); + assert.equal(document.descriptor.key, 'think.memory-document.v1'); +}); + +test('native Think documents round-trip a typed aggregate', () => { + const expected = Object.freeze({ + id: 'entry:test', + kind: 'capture', + text: 'bounded v19 document', + createdAt: '2026-07-28T00:00:00.000Z', + }); + const encoded = encodeNativeDocument('memory-object', expected); + const decoded = decodeNativeDocument(encoded, 'memory-object'); + + assert.deepEqual(decoded, expected); + assert.equal(decodeNativeDocument(encoded, 'memory-index'), null); +}); + +test('native coordinate reader reuses one bounded optic for index pages', async () => { + const indexBytes = encodeNativeDocument('memory-index', { + total: 65, + headPage: 1, + }); + const pageBytes = encodeNativeDocument('memory-index-page', { + entries: [{ id: 'entry:test', kind: 'capture' }], + }); + const { optic, reads } = coordinateReaderFixture(indexBytes, pageBytes); + const reader = new NativeMemoryCoordinateReader(optic); + + assert.deepEqual(await reader.memoryIndex('index:test'), indexBytes); + assert.deepEqual(await reader.memoryIndexPage('page:test'), pageBytes); + assert.equal(await reader.memoryIndexPage('page:missing'), null); + assert.deepEqual(reads, [ + { subject: 'index:test', key: INDEX_DOCUMENT_KEY }, + { subject: 'page:test', key: INDEX_PAGE_DOCUMENT_KEY }, + { subject: 'page:missing', key: INDEX_PAGE_DOCUMENT_KEY }, + ]); +}); + +function coordinateReaderFixture(indexBytes, pageBytes) { + const reads = []; + const values = new Map([ + [`index:test\0${INDEX_DOCUMENT_KEY}`, indexBytes], + [`page:test\0${INDEX_PAGE_DOCUMENT_KEY}`, { + toUint8Array: () => pageBytes, + }], + ]); + const optic = { + node(subject) { + return { + prop(key) { + return { + read() { + reads.push({ subject, key }); + const value = values.get(`${subject}\0${key}`); + return Promise.resolve({ + exists: value !== undefined, + value, + }); + }, + }; + }, + }; + }, + }; + return { optic, reads }; +} diff --git a/test/ports/no-full-materialization.test.js b/test/ports/no-full-materialization.test.js index 5f0eb753..b206287c 100644 --- a/test/ports/no-full-materialization.test.js +++ b/test/ports/no-full-materialization.test.js @@ -3,10 +3,37 @@ import { readFileSync, readdirSync, statSync } from 'node:fs'; import path from 'node:path'; import test from 'node:test'; -import { stringifyJson } from '../../src/json.js'; -import { CAPTURE_READ_MODEL_ID } from '../../src/store/constants.js'; -import { rememberThoughtsForRead } from '../../src/store/queries.js'; -import { ambientReadModelId } from '../../src/store/read-model.js'; +const FORBIDDEN_RUNTIME_PATTERNS = Object.freeze([ + ['full node materialization', /\.getNodes\(\)/u], + ['full edge materialization', /\.getEdges\(\)/u], + ['wildcard query facade', /\.query\(\)\s*\.match\(/u], + ['legacy Think record key', /think\.record\.v1/u], + ['private git-warp distribution import', /@git-stunts\/git-warp\/dist\//u], +]); + +test('production source contains no legacy graph facade or full materialization path', () => { + const srcDir = new URL('../../src/', import.meta.url).pathname; + const violations = []; + for (const file of collectJsFiles(srcDir)) { + const content = readFileSync(file, 'utf8'); + const relative = path.relative(srcDir, file); + for (const [name, pattern] of FORBIDDEN_RUNTIME_PATTERNS) { + if (pattern.test(withoutComments(content))) { + violations.push(`${relative}: ${name}`); + } + } + } + assert.deepEqual(violations, []); +}); + +test('rejected compatibility modules are absent', () => { + const storeDir = new URL('../../src/store/', import.meta.url).pathname; + const files = new Set(readdirSync(storeDir)); + assert.equal(files.has('git-warp-v19.js'), false); + assert.equal(files.has('think-warp-sdk.js'), false); + assert.equal(files.has('v19-record.js'), false); + assert.equal(files.has('read-model.js'), false); +}); function collectJsFiles(dir) { const files = []; @@ -21,197 +48,8 @@ function collectJsFiles(dir) { return files; } -test('no source file calls getNodes() or getEdges() for full graph materialization', () => { - const srcDir = new URL('../../src/', import.meta.url).pathname; - const files = collectJsFiles(srcDir); - const violations = []; - - for (const file of files) { - const content = readFileSync(file, 'utf8'); - const relPath = path.relative(path.join(srcDir, '..'), file); - - // Match .getNodes() or .getEdges() but not in comments - const lines = content.split('\n'); - for (let i = 0; i < lines.length; i++) { - const line = lines[i].trim(); - if (line.startsWith('//') || line.startsWith('*')) { continue; } - if (/\.getNodes\(\)/.test(line)) { - violations.push(`${relPath}:${i + 1}: ${line.trim()}`); - } - if (/\.getEdges\(\)/.test(line)) { - violations.push(`${relPath}:${i + 1}: ${line.trim()}`); - } - } - } - - assert.deepEqual( - violations, - [], - `Found ${violations.length} full-materialization anti-pattern(s):\n${violations.join('\n')}` - ); -}); - -test('default ambient remember filters bounded recent captures instead of graph queries', async () => { - const cwd = '/tmp/think-alpha-project'; - const matching = capture('entry:1780000002000-alpha', 'alpha project should stay fast', { - ambientCwd: cwd, - }); - const other = capture('entry:1780000001000-beta', 'lunch note should not rank', { - ambientCwd: '/tmp/think-beta-project', - }); - const read = createBoundedRead([matching, other], { ambientIndexes: [{ key: 'ambientCwd', value: cwd, entries: [matching] }] }); - - const remembered = await rememberThoughtsForRead(read, { cwd, limit: 5 }); - - assert.deepEqual( - remembered.matches.map((match) => match.entryId), - [matching.id], - 'Expected ambient remember to score bounded recent captures and avoid graph-wide capture scans.' - ); - assert.equal(read.queryCalls, 0, 'Expected default ambient remember not to call read.view.query().'); -}); - -test('default ambient remember uses one fast capture record property when available', async () => { - const cwd = '/tmp/think-fast-project'; - const matching = capture('entry:1780000002000-fast', 'fast project record should be enough', { - ambientCwd: cwd, - }); - const other = capture('entry:1780000001000-other', 'unrelated lunch note should not rank', { - ambientCwd: '/tmp/think-other-project', - }); - const read = createFastRecordRead([matching, other]); - - const remembered = await rememberThoughtsForRead(read, { cwd, limit: 5 }); - - assert.deepEqual( - remembered.matches.map((match) => match.entryId), - [matching.id], - 'Expected ambient remember to filter self-contained fast capture records.' - ); - assert.deepEqual( - read.propReads, - [{ nodeId: CAPTURE_READ_MODEL_ID, key: 'fastCaptureRecordsJson' }], - 'Expected default remember to read only the fast capture record property.' - ); - assert.equal(read.nodePropCalls, 0, 'Expected fast record recall not to hydrate capture nodes.'); - assert.equal(read.queryCalls, 0, 'Expected fast record recall not to call read.view.query().'); -}); - -test('explicit remember filters bounded recent refs instead of bootstrapping keyword scans', async () => { - const matching = capture('entry:1780000002000-warp', 'warp receipts need a bounded recall path'); - const other = capture('entry:1780000001000-lunch', 'lunch notes are unrelated'); - const read = createBoundedRead([matching, other]); - - const remembered = await rememberThoughtsForRead(read, { query: 'warp receipts', limit: 5 }); - - assert.deepEqual( - remembered.matches.map((match) => match.entryId), - [matching.id], - 'Expected explicit remember to search the bounded recent read model without a keyword:* scan.' - ); - assert.equal(read.queryCalls, 0, 'Expected explicit remember not to call read.view.query().'); -}); - -function capture(id, text, props = {}) { - return Object.freeze({ - id, - text, - props: Object.freeze({ - kind: 'capture', - writerId: 'test', - createdAt: new Date(Number(id.split(':')[1].split('-')[0])).toISOString(), - sortKey: id.slice('entry:'.length), - ...props, - }), - }); -} - -function createBoundedRead(entries, { ambientIndexes = [] } = {}) { - const propsById = new Map(entries.map((entry) => [entry.id, entry.props])); - const textById = new Map(entries.map((entry) => [entry.id, entry.text])); - const read = { queryCalls: 0, repoDir: '/tmp/think-bounded-read' }; - - read.view = { - getNodeProps(nodeId) { - if (nodeId === CAPTURE_READ_MODEL_ID) { - return readModelProps(entries); - } - - for (const index of ambientIndexes) { - if (nodeId === ambientReadModelId(index.key, index.value)) { - return ambientReadModelProps(index); - } - } - - return propsById.get(nodeId) ?? null; - }, - query() { - read.queryCalls += 1; - throw new Error('Expected bounded recall to avoid read.view.query().'); - }, - }; - read.readContent = (nodeId) => new TextEncoder().encode(textById.get(nodeId) ?? ''); - return read; -} - -function createFastRecordRead(entries) { - const read = { - nodePropCalls: 0, - propReads: [], - queryCalls: 0, - repoDir: '/tmp/think-fast-read', - }; - - read.readNodeProp = (nodeId, key) => { - read.propReads.push({ nodeId, key }); - if (nodeId === CAPTURE_READ_MODEL_ID && key === 'fastCaptureRecordsJson') { - return stringifyJson(entries.map(entryRecord)); - } - return undefined; - }; - read.view = { - getNodeProps() { - read.nodePropCalls += 1; - throw new Error('Expected fast record recall not to hydrate graph nodes.'); - }, - query() { - read.queryCalls += 1; - throw new Error('Expected bounded recall to avoid read.view.query().'); - }, - }; - return read; -} - -function readModelProps(entries) { - return { - kind: 'capture_read_model', - latestCaptureId: entries[0]?.id ?? null, - totalCaptures: entries.length, - recentCaptureRefsJson: stringifyJson(entries.map(entryRef)), - }; -} - -function ambientReadModelProps(index) { - return { - kind: 'ambient_capture_read_model', - ambientKey: index.key, - ambientValue: index.value, - recentCaptureRefsJson: stringifyJson(index.entries.map(entryRef)), - }; -} - -function entryRef(entry) { - return { - id: entry.id, - createdAt: entry.props.createdAt, - sortKey: entry.props.sortKey, - }; -} - -function entryRecord(entry) { - return { - ...entryRef(entry), - ...entry.props, - text: entry.text, - }; +function withoutComments(source) { + return source + .replace(/\/\*[\s\S]*?\*\//gu, '') + .replace(/^\s*\/\/.*$/gmu, ''); } diff --git a/test/ports/recent-runtime.test.js b/test/ports/recent-runtime.test.js index acd97ded..7c6672bf 100644 --- a/test/ports/recent-runtime.test.js +++ b/test/ports/recent-runtime.test.js @@ -1,178 +1,94 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { stringifyJson } from '../../src/json.js'; -import { CAPTURE_READ_MODEL_ID } from '../../src/store/constants.js'; import { getStoredEntry, listRecentStoredEntries, readNodeText, } from '../../src/store/runtime.js'; -test('listRecentStoredEntries does not traverse older edges beyond the requested limit', async () => { - const read = createFakeChronologyRead(['entry:newest', 'entry:middle', 'entry:oldest']); +test('listRecentStoredEntries requests only the bounded native capture window', async () => { + const read = createNativeRead([ + capture('entry:newest'), + capture('entry:middle'), + capture('entry:oldest'), + ]); const entries = await listRecentStoredEntries(read, { limit: 1 }); - assert.deepEqual(entries.map((entry) => entry.id), ['entry:newest']); - assert.equal(read.olderLookups, 0, 'Expected limit=1 not to traverse the older chain.'); - assert.equal(read.queryCalls, 0, 'Expected recent reads to avoid graph queries entirely.'); + assert.deepEqual(entries.map(entry => entry.id), ['entry:newest']); + assert.deepEqual(read.indexReads, [{ kind: 'capture', limit: 1 }]); + assert.deepEqual(read.documentReads, []); }); -test('listRecentStoredEntries hydrates only the requested sorted capture window', async () => { - const read = createFakeChronologyRead(['entry:newest', 'entry:middle', 'entry:oldest']); +test('listRecentStoredEntries hydrates the capped page documents without exact reads', async () => { + const read = createNativeRead([ + capture('entry:newest'), + capture('entry:middle'), + capture('entry:oldest'), + ]); const entries = await listRecentStoredEntries(read, { limit: 2 }); - assert.deepEqual(entries.map((entry) => entry.id), ['entry:newest', 'entry:middle']); - assert.equal(read.olderLookups, 0, 'Expected recent reads to sort capture props without traversing older edges.'); - assert.equal(read.queryCalls, 0, 'Expected recent reads to hydrate exact indexed capture IDs.'); + assert.deepEqual(entries.map(entry => entry.id), ['entry:newest', 'entry:middle']); + assert.deepEqual(read.indexReads, [{ kind: 'capture', limit: 2 }]); + assert.deepEqual(read.documentReads, []); }); -test('getStoredEntry keeps the public readContent fallback for content oids', async () => { - const props = createCaptureProps({ _content: 'oid:test-content' }); - const read = createTextReadHandle({ - readContent() { - return new TextEncoder().encode('fallback text'); - }, - }); - - const entry = await getStoredEntry(read, 'entry:fallback-content', props); +test('getStoredEntry decodes one exact native memory document', async () => { + const expected = capture('entry:exact', { text: 'native document text' }); + const read = createNativeRead([expected]); - assert.equal(entry.text, 'fallback text'); -}); + const entry = await getStoredEntry(read, expected.id); -test('getStoredEntry fails when content oid has no readable content source', async () => { - const props = createCaptureProps({ _content: 'oid:missing-content' }); - const read = createTextReadHandle(); - - await assert.rejects( - () => getStoredEntry(read, 'entry:missing-content', props), - (error) => { - assert.equal(error.name, 'ContentUnavailableError'); - assert.equal(error.code, 'CONTENT_UNAVAILABLE'); - assert.match(error.message, /entry:missing-content/); - return true; - }, - ); + assert.equal(entry.text, 'native document text'); + assert.deepEqual(read.documentReads, [expected.id]); }); -test('readNodeText skips marked content readers for nodes without content oids', async () => { - const read = createTextReadHandle({ - readContentRequiresContentOid: true, - readContent() { - throw new Error('Expected empty nodes not to read external content.'); - }, - }); +test('getStoredEntry returns null when the native document is absent', async () => { + const read = createNativeRead([]); - const text = await readNodeText(read, 'entry:empty-content', createCaptureProps()); - - assert.equal(text, ''); + assert.equal(await getStoredEntry(read, 'entry:missing'), null); + assert.deepEqual(read.documentReads, ['entry:missing']); }); -function createFakeChronologyRead(ids) { - const propsById = createFakePropsById(ids); - const textById = new Map(ids.map((id) => [id, `text for ${id}`])); - const read = { olderLookups: 0, queryCalls: 0 }; - - read.view = createFakeChronologyView(ids, propsById, read); - read.readContent = (nodeId) => new TextEncoder().encode(textById.get(nodeId) ?? ''); - return read; -} +test('readNodeText reads inline text from the native document only', async () => { + const expected = capture('entry:text', { text: 'inline text' }); + const read = createNativeRead([expected]); -function createTextReadHandle(options = {}) { - return { - view: { - getNodeProps() { - throw new Error('Expected test to pass props directly.'); - }, - }, - ...options, - }; -} + assert.equal(await readNodeText(read, expected.id), 'inline text'); + assert.deepEqual(read.documentReads, [expected.id]); +}); -function createCaptureProps(overrides = {}) { - return { +function capture(id, overrides = {}) { + return Object.freeze({ + id, kind: 'capture', writerId: 'writer:test', createdAt: new Date(1770000000000).toISOString(), - sortKey: '1770000000000', + sortKey: id, + text: `text for ${id}`, ...overrides, - }; -} - -function createFakePropsById(ids) { - return new Map(ids.map((id, index) => [id, createCaptureProps({ - createdAt: new Date(1770000000000 - index).toISOString(), - sortKey: String(1770000000000 - index), - })])); + }); } -function createFakeChronologyView(ids, propsById, read) { - return { - query() { - read.queryCalls += 1; - return createFakeChronologyQuery(ids, propsById, read); - }, - getNodeProps(nodeId) { - if (nodeId === CAPTURE_READ_MODEL_ID) { - return { - kind: 'capture_read_model', - latestCaptureId: ids[0] ?? null, - totalCaptures: ids.length, - recentCaptureRefsJson: stringifyJson(ids.map((id) => ({ - id, - createdAt: propsById.get(id).createdAt, - sortKey: propsById.get(id).sortKey, - }))), - }; - } - return propsById.get(nodeId) ?? null; - }, +function createNativeRead(documents) { + const documentsById = new Map(documents.map(document => [document.id, document])); + const read = { + documentReads: [], + indexReads: [], + repoDir: '/tmp/think-native-read', }; -} - -function createFakeChronologyQuery(ids, propsById, read) { - const state = { nodeId: null, label: null, criteria: null }; - const query = { - match(nodeId) { - state.nodeId = nodeId; - return query; - }, - where(criteria) { - state.criteria = criteria; - return query; - }, - outgoing(label) { - state.label = label; - return query; - }, - run() { - return runFakeChronologyQuery(state, ids, propsById, read); - }, + read.listDocumentsByKind = (kind, { limit }) => { + read.indexReads.push({ kind, limit }); + return documents.filter(document => document.kind === kind).slice(0, limit); }; - return query; -} - -function runFakeChronologyQuery(state, ids, propsById, read) { - if (state.criteria?.kind === 'capture') { - return { nodes: ids.map((id) => ({ id, props: propsById.get(id) })) }; - } - - if (state.label === 'latest_capture') { - return { nodes: ids.length > 0 ? [{ id: ids[0] }] : [] }; - } - - if (state.label === 'older') { - read.olderLookups += 1; - return { nodes: getFakeOlderNode(state.nodeId, ids) }; - } - - return { nodes: [] }; -} - -function getFakeOlderNode(nodeId, ids) { - const currentIndex = ids.indexOf(nodeId); - const olderId = currentIndex === -1 ? null : ids[currentIndex + 1] ?? null; - return olderId ? [{ id: olderId }] : []; + read.readDocument = (nodeId) => { + read.documentReads.push(nodeId); + return documentsById.get(nodeId) ?? null; + }; + read.memory = { + memoryDocument: () => null, + }; + return read; } diff --git a/test/ports/repair-v17-mind.test.js b/test/ports/repair-v17-mind.test.js deleted file mode 100644 index 44d430bf..00000000 --- a/test/ports/repair-v17-mind.test.js +++ /dev/null @@ -1,291 +0,0 @@ -import assert from 'node:assert/strict'; -import path from 'node:path'; -import test from 'node:test'; - -import { - createBackupRef, - formatBackupTimestamp, - normalizeContentAnchorTreeEntries, - parseRepairArgs, - repairV17Mind, - resolveRepairTarget, -} from '../../scripts/repair-v17-mind.mjs'; - -const OLD_CHECKPOINT = '1111111111111111111111111111111111111111'; -const UPGRADED_CHECKPOINT = '2222222222222222222222222222222222222222'; -const FRESH_CHECKPOINT = '3333333333333333333333333333333333333333'; -const ZERO_OID = '0000000000000000000000000000000000000000'; - -test('repair args resolve a named mind under ~/.think', () => { - const args = parseRepairArgs(['--mind', 'claude', '--dry-run', '--json']); - const target = resolveRepairTarget(args, { - homeDir: '/Users/example', - }); - - assert.equal(target.graph, 'think'); - assert.equal(target.mind, 'claude'); - assert.equal(target.repoDir, path.join('/Users/example', '.think', 'claude')); -}); - -test('repair args map the default mind to ~/.think/repo', () => { - const args = parseRepairArgs(['--mind', 'default']); - const target = resolveRepairTarget(args, { - homeDir: '/Users/example', - }); - - assert.equal(target.mind, 'default'); - assert.equal(target.repoDir, path.join('/Users/example', '.think', 'repo')); -}); - -test('repair args reject ambiguous mind and repo targets', () => { - const args = parseRepairArgs(['--mind', 'claude', '--repo', '/tmp/mind']); - - assert.throws( - () => resolveRepairTarget(args), - /Use either --mind or --repo/ - ); -}); - -test('content anchor normalization preserves blob anchors as blob entries', async () => { - const blobOid = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; - const treeOid = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'; - const frontierOid = 'cccccccccccccccccccccccccccccccccccccccc'; - const objectTypes = new Map([ - [blobOid, 'blob'], - [treeOid, 'tree'], - ]); - - const normalized = await normalizeContentAnchorTreeEntries([ - `040000 tree ${blobOid}\t_content_${blobOid}`, - `040000 tree ${treeOid}\t_content_${treeOid}`, - `100644 blob ${frontierOid}\tfrontier.cbor`, - ], oid => objectTypes.get(oid)); - - assert.deepEqual(normalized, [ - `100644 blob ${blobOid}\t_content_${blobOid}`, - `040000 tree ${treeOid}\t_content_${treeOid}`, - `100644 blob ${frontierOid}\tfrontier.cbor`, - ]); -}); - -test('createBackupRef creates a dated pre-v17 ref without overwriting an existing ref', () => { - const now = new Date('2026-05-12T16:25:00.000Z'); - const firstRef = 'refs/warp/think/checkpoints/pre-v17-upgrade-20260512-162500'; - const existingRefs = new Set([firstRef]); - const updates = []; - const runner = createGitRunner({ - existingRefs, - updates, - }); - - const backupRef = createBackupRef({ - checkpointSha: OLD_CHECKPOINT, - graph: 'think', - now, - repoDir: '/tmp/think-mind', - runner, - }); - - assert.equal(formatBackupTimestamp(now), '20260512-162500'); - assert.equal(backupRef, 'refs/warp/think/checkpoints/pre-v17-upgrade-20260512-162500-02'); - assert.deepEqual(updates, [{ - newValue: OLD_CHECKPOINT, - oldValue: ZERO_OID, - ref: backupRef, - }]); -}); - -test('repairV17Mind dry-run reports needed repair without mutating refs', async () => { - const events = []; - const runner = createGitRunner({ events }); - let dryRunCount = 0; - - const result = await repairV17Mind({ - dryRun: true, - graph: 'think', - help: false, - json: true, - mind: null, - repo: '/tmp/think-mind', - }, { - runner, - runUpgrade: ({ dryRun }) => { - assert.equal(dryRun, true); - dryRunCount += 1; - return upgradeWouldRun(); - }, - materialize: () => { - throw new Error('dry-run must not materialize'); - }, - }); - - assert.equal(result.ok, true); - assert.equal(result.dryRun, true); - assert.equal(result.wouldRepair, true); - assert.equal(result.backupRef, null); - assert.equal(dryRunCount, 1); - assert.ok(!events.includes('backup'), 'dry-run must not create backup refs'); -}); - -test('repairV17Mind backs up before upgrade and verifies the repaired graph', async () => { - const events = []; - const updates = []; - const runner = createGitRunner({ - events, - updates, - }); - let dryRunCount = 0; - - const result = await repairV17Mind({ - dryRun: false, - graph: 'think', - help: false, - json: true, - mind: null, - repo: '/tmp/think-mind', - }, { - now: new Date('2026-05-12T16:25:00.000Z'), - runner, - runCheck: () => ({ - checkpoint: { - sha: FRESH_CHECKPOINT, - }, - status: { - patchesSinceCheckpoint: 0, - }, - }), - runDoctor: () => ({ - findings: [ - { id: 'repo-accessible', status: 'ok' }, - { id: 'hooks-installed', status: 'warn' }, - ], - summary: { - fail: 0, - warn: 1, - }, - }), - runUpgrade: ({ dryRun }) => { - if (dryRun) { - dryRunCount += 1; - return dryRunCount === 1 ? upgradeWouldRun() : upgradeAlreadyCurrent(); - } - - events.push('upgrade'); - return { - checkpointRef: 'refs/warp/think/checkpoints/head', - currentSchema: 5, - graphName: 'think', - previousCheckpointSha: OLD_CHECKPOINT, - previousSchema: 4, - status: 'upgraded', - upgradedCheckpointSha: UPGRADED_CHECKPOINT, - }; - }, - materialize: () => { - events.push('materialize'); - return { - checkpoint: FRESH_CHECKPOINT, - graph: 'think', - }; - }, - }); - - const backupRef = 'refs/warp/think/checkpoints/pre-v17-upgrade-20260512-162500'; - - assert.equal(result.ok, true); - assert.equal(result.changed, true); - assert.equal(result.beforeCheckpoint, OLD_CHECKPOINT); - assert.equal(result.afterCheckpoint, FRESH_CHECKPOINT); - assert.equal(result.backupRef, backupRef); - assert.equal(result.upgrade.before, 'needed'); - assert.equal(result.upgrade.after, 'already-current'); - assert.equal(result.check.patchesSinceCheckpoint, 0); - assert.equal(result.doctor.failures, 0); - assert.equal(result.doctor.warnings.length, 1); - assert.deepEqual(updates, [{ - newValue: OLD_CHECKPOINT, - oldValue: ZERO_OID, - ref: backupRef, - }]); - assert.deepEqual(events, ['backup', 'upgrade', 'materialize']); -}); - -function upgradeWouldRun() { - return { - checkpointRef: 'refs/warp/think/checkpoints/head', - currentSchema: 5, - graphName: 'think', - previousCheckpointSha: OLD_CHECKPOINT, - previousSchema: 4, - status: 'would-upgrade', - upgradedCheckpointSha: null, - }; -} - -function upgradeAlreadyCurrent() { - return { - checkpointRef: 'refs/warp/think/checkpoints/head', - currentSchema: 5, - graphName: 'think', - previousCheckpointSha: FRESH_CHECKPOINT, - previousSchema: 5, - status: 'already-current', - upgradedCheckpointSha: FRESH_CHECKPOINT, - }; -} - -function createGitRunner({ - events = [], - existingRefs = new Set(), - updates = [], -} = {}) { - return (_command, args, _options = {}) => { - const gitArgs = args.slice(2); - const subcommand = gitArgs[0]; - - if (subcommand === 'rev-parse' && gitArgs.includes('--git-dir')) { - return commandResult({ stdout: '.git\n' }); - } - - if (subcommand === 'rev-parse' && gitArgs.includes('--verify')) { - return commandResult({ stdout: `${OLD_CHECKPOINT}\n` }); - } - - if (subcommand === 'show-ref') { - const ref = gitArgs.at(-1); - return commandResult({ - status: existingRefs.has(ref) ? 0 : 1, - }); - } - - if (subcommand === 'update-ref') { - const ref = gitArgs[2]; - const newValue = gitArgs[3]; - const oldValue = gitArgs[4]; - events.push('backup'); - updates.push({ newValue, oldValue, ref }); - existingRefs.add(ref); - return commandResult(); - } - - if (subcommand === 'cat-file') { - return commandResult({ stdout: 'blob\n' }); - } - - throw new Error(`Unexpected git command: ${gitArgs.join(' ')}`); - }; -} - -function commandResult({ - status = 0, - stdout = '', - stderr = '', -} = {}) { - return Object.freeze({ - args: [], - command: 'git', - status, - stderr, - stdout, - }); -}