From a5e1a4a35d01e09cbfd2c503370c95a8c2490867 Mon Sep 17 00:00:00 2001 From: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Date: Thu, 16 Apr 2026 16:01:04 +0300 Subject: [PATCH 01/56] Bump CLI dependencies (#5383) --- .changeset/cozy-wombats-burn.md | 5 +++++ packages/cli/lib/util.js | 4 ++-- packages/cli/package.json | 3 +-- 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 .changeset/cozy-wombats-burn.md diff --git a/.changeset/cozy-wombats-burn.md b/.changeset/cozy-wombats-burn.md new file mode 100644 index 0000000000..e04a363fcd --- /dev/null +++ b/.changeset/cozy-wombats-burn.md @@ -0,0 +1,5 @@ +--- +"@apostrophecms/cli": patch +--- + +Bump and clean up depdendencies. diff --git a/packages/cli/lib/util.js b/packages/cli/lib/util.js index f531f47e77..566b3e75fc 100644 --- a/packages/cli/lib/util.js +++ b/packages/cli/lib/util.js @@ -7,7 +7,7 @@ const util = {}; const _ = require('lodash'); const cliVersion = require('../package.json').version; const confUtils = require('./conf-utils'); -const packageJson = require('package-json'); +const { execSync } = require('child_process'); const semver = require('semver'); const ora = require('ora'); const { spawn } = require('child_process'); @@ -218,7 +218,7 @@ util.spawnWithSpinner = async function (command, options = { async function checkIfUpdated () { try { // Get the latest published version number. - const { version: latest } = await packageJson('@apostrophecms/cli'); + const latest = execSync('npm show @apostrophecms/cli version', { encoding: 'utf8' }).trim(); // Check if they've been notified for this version already. If so, bail out. const latestChecked = await confUtils.getConf('versionNotified'); diff --git a/packages/cli/package.json b/packages/cli/package.json index 25c13dcde7..ff9c72b249 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -23,12 +23,11 @@ "conf": "^6.2.4", "lodash": "^4.18.1", "ora": "^5.4.1", - "package-json": "^6.5.0", "pkginfo": "^0.4.1", "prompts": "^2.3.2", "semver": "^7.3.2", "shell-quote": "^1.8.1", - "shelljs": "~0.3.x", + "shelljs": "^0.10.0", "sync-request": "^6.0.0", "uuid": "^8.0.0" }, From 9f458b5d9544d33239a0971b0050fff8bffe028a Mon Sep 17 00:00:00 2001 From: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Date: Thu, 16 Apr 2026 16:38:22 +0300 Subject: [PATCH 02/56] Fix choices IDs (#5379) * Fix choices IDs * Cleanup dots from ID values * Update changelog Co-authored-by: Robert Means --------- Co-authored-by: Robert Means --- .changeset/clean-actors-laugh.md | 5 +++++ .../schema/ui/apos/logic/AposInputCheckboxes.js | 2 +- .../@apostrophecms/schema/ui/apos/logic/AposInputRadio.js | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .changeset/clean-actors-laugh.md diff --git a/.changeset/clean-actors-laugh.md b/.changeset/clean-actors-laugh.md new file mode 100644 index 0000000000..5d78fab358 --- /dev/null +++ b/.changeset/clean-actors-laugh.md @@ -0,0 +1,5 @@ +--- +"apostrophe": patch +--- + +Fix illegal HTML id attribute values generated by the admin UI diff --git a/packages/apostrophe/modules/@apostrophecms/schema/ui/apos/logic/AposInputCheckboxes.js b/packages/apostrophe/modules/@apostrophecms/schema/ui/apos/logic/AposInputCheckboxes.js index 45a8a4c6d0..73fa374d04 100644 --- a/packages/apostrophe/modules/@apostrophecms/schema/ui/apos/logic/AposInputCheckboxes.js +++ b/packages/apostrophe/modules/@apostrophecms/schema/ui/apos/logic/AposInputCheckboxes.js @@ -11,7 +11,7 @@ export default { }, methods: { getChoiceId(uid, value) { - return (uid + JSON.stringify(value)).replace(/\s+/g, ''); + return (uid + String(value)).replace(/[\s"'.]/g, ''); }, watchValue () { this.error = this.modelValue.error; diff --git a/packages/apostrophe/modules/@apostrophecms/schema/ui/apos/logic/AposInputRadio.js b/packages/apostrophe/modules/@apostrophecms/schema/ui/apos/logic/AposInputRadio.js index 97dffa11ed..f7d1b6f6e9 100644 --- a/packages/apostrophe/modules/@apostrophecms/schema/ui/apos/logic/AposInputRadio.js +++ b/packages/apostrophe/modules/@apostrophecms/schema/ui/apos/logic/AposInputRadio.js @@ -8,7 +8,7 @@ export default { mixins: [ AposInputMixin, AposInputChoicesMixin ], methods: { getChoiceId(uid, value) { - return (uid + JSON.stringify(value)).replace(/\s+/g, ''); + return (uid + String(value)).replace(/[\s"'.]/g, ''); }, validate(value) { const validValue = this.choices.some((choice) => choice.value === value); From b9b32bdd446add927c75d85e44d0a26814ba62b0 Mon Sep 17 00:00:00 2001 From: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Date: Fri, 17 Apr 2026 13:30:56 +0300 Subject: [PATCH 03/56] Native browser shortcuts work again (#5384) --- .changeset/full-symbols-obey.md | 5 +++++ .../modules/@apostrophecms/area/index.js | 15 ++++++++++----- .../ui/apos/components/TheAposCommandMenu.vue | 12 +++++++++++- 3 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 .changeset/full-symbols-obey.md diff --git a/.changeset/full-symbols-obey.md b/.changeset/full-symbols-obey.md new file mode 100644 index 0000000000..903395756e --- /dev/null +++ b/.changeset/full-symbols-obey.md @@ -0,0 +1,5 @@ +--- +"apostrophe": patch +--- + +Keyboard shortcuts for widget operations (copy, cut, paste, duplicate, remove) no longer block the browser's native clipboard behavior when no widget is focused. Previously, selecting and copying text on a page while logged in was prevented by the admin UI intercepting those shortcuts unconditionally. diff --git a/packages/apostrophe/modules/@apostrophecms/area/index.js b/packages/apostrophe/modules/@apostrophecms/area/index.js index 47b83516f1..a96d8248c8 100644 --- a/packages/apostrophe/modules/@apostrophecms/area/index.js +++ b/packages/apostrophe/modules/@apostrophecms/area/index.js @@ -19,7 +19,8 @@ module.exports = { action: { type: 'command-menu-area-cut-widget' }, - shortcut: 'Ctrl+X Meta+X' + shortcut: 'Ctrl+X Meta+X', + requireWidgetFocus: true }, [`${self.__meta.name}:copy-widget`]: { type: 'item', @@ -27,7 +28,8 @@ module.exports = { action: { type: 'command-menu-area-copy-widget' }, - shortcut: 'Ctrl+C Meta+C' + shortcut: 'Ctrl+C Meta+C', + requireWidgetFocus: true }, [`${self.__meta.name}:paste-widget`]: { type: 'item', @@ -35,7 +37,8 @@ module.exports = { action: { type: 'command-menu-area-paste-widget' }, - shortcut: 'Ctrl+V Meta+V' + shortcut: 'Ctrl+V Meta+V', + requireWidgetFocus: true }, [`${self.__meta.name}:duplicate-widget`]: { type: 'item', @@ -43,7 +46,8 @@ module.exports = { action: { type: 'command-menu-area-duplicate-widget' }, - shortcut: 'Ctrl+Shift+D Meta+Shift+D' + shortcut: 'Ctrl+Shift+D Meta+Shift+D', + requireWidgetFocus: true }, [`${self.__meta.name}:remove-widget`]: { type: 'item', @@ -51,7 +55,8 @@ module.exports = { action: { type: 'command-menu-area-remove-widget' }, - shortcut: 'Backspace' + shortcut: 'Backspace', + requireWidgetFocus: true } }, modal: { diff --git a/packages/apostrophe/modules/@apostrophecms/command-menu/ui/apos/components/TheAposCommandMenu.vue b/packages/apostrophe/modules/@apostrophecms/command-menu/ui/apos/components/TheAposCommandMenu.vue index 6712ec169e..6003a5830d 100644 --- a/packages/apostrophe/modules/@apostrophecms/command-menu/ui/apos/components/TheAposCommandMenu.vue +++ b/packages/apostrophe/modules/@apostrophecms/command-menu/ui/apos/components/TheAposCommandMenu.vue @@ -10,6 +10,7 @@ import { mapActions, mapState } from 'pinia'; import AposThemeMixin from 'Modules/@apostrophecms/ui/mixins/AposThemeMixin'; import { useModalStore } from 'Modules/@apostrophecms/ui/stores/modal'; +import { useWidgetStore } from 'Modules/@apostrophecms/ui/stores/widget'; export default { name: 'TheAposCommandMenu', @@ -50,7 +51,13 @@ export default { .flatMap(command => { return command.shortcut .split(' ') - .map(shortcut => [ shortcut.toUpperCase(), command.action ]); + .map(shortcut => [ + shortcut.toUpperCase(), + { + ...command.action, + requireWidgetFocus: command.requireWidgetFocus || false + } + ]); }); }) ); @@ -111,6 +118,9 @@ export default { ? keys.slice('SHIFT+'.length) : keys]; if (action) { + if (action.requireWidgetFocus && !useWidgetStore().focusedWidget) { + return; + } event.preventDefault(); apos.bus.$emit(action.type, action.payload); return; From 08845c5f23d95ead552ab9743a878daf997793a1 Mon Sep 17 00:00:00 2001 From: Robert Means Date: Wed, 22 Apr 2026 08:00:35 -0400 Subject: [PATCH 04/56] Pro 8838 charset (#5385) * Removes encoding option and comments hardcoded encoding meta * add changeset * Response to first comments --- .changeset/fifty-hornets-follow.md | 7 +++++++ packages/apostrophe/modules/@apostrophecms/i18n/index.js | 9 +-------- .../@apostrophecms/template/views/outerLayoutBase.html | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 .changeset/fifty-hornets-follow.md diff --git a/.changeset/fifty-hornets-follow.md b/.changeset/fifty-hornets-follow.md new file mode 100644 index 0000000000..52da86bff8 --- /dev/null +++ b/.changeset/fifty-hornets-follow.md @@ -0,0 +1,7 @@ +--- +"apostrophe": minor +--- + +- Removed duplicate tag from `outerLayoutBase.html` +- Standardized charset to utf-8 (removed legacy configuration option) +- Altered unused/legacy i18n template helper to return `utf-8` (BC) diff --git a/packages/apostrophe/modules/@apostrophecms/i18n/index.js b/packages/apostrophe/modules/@apostrophecms/i18n/index.js index 96de6ad5ac..66d0aa9f8d 100644 --- a/packages/apostrophe/modules/@apostrophecms/i18n/index.js +++ b/packages/apostrophe/modules/@apostrophecms/i18n/index.js @@ -29,10 +29,6 @@ // in the same language as the website content. // Example: `defaultAdminLocale: 'fr'`. // -// ### `encoding` -// -// Defaults to `'utf-8'`. You almost certainly do not want to change this. -// // ### `slugDirection` // // Controls the default `direction` value of slug schema. Can be `ltr`, `rtl` or @@ -81,8 +77,6 @@ module.exports = { }, // If true, slugifying will strip accents from Latin characters stripUrlAccents: false, - // You almost certainly do not want to change this - encoding: 'utf-8', slugDirection: 'ltr' }, async init(self) { @@ -166,7 +160,6 @@ module.exports = { await self.i18next.init(i18nextOptions); self.addInitialResources(); self.enableBrowserData(); - self.encoding = self.options.encoding; }, handlers(self) { return { @@ -1369,7 +1362,7 @@ module.exports = { helpers(self) { return { encoding() { - return self.encoding; + return 'utf-8'; } }; } diff --git a/packages/apostrophe/modules/@apostrophecms/template/views/outerLayoutBase.html b/packages/apostrophe/modules/@apostrophecms/template/views/outerLayoutBase.html index 3ef25932f2..12c60de95f 100644 --- a/packages/apostrophe/modules/@apostrophecms/template/views/outerLayoutBase.html +++ b/packages/apostrophe/modules/@apostrophecms/template/views/outerLayoutBase.html @@ -2,6 +2,7 @@ {% block encoding %} + {# Per spec (https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/meta#charset) the only allowed value for this attribute is `utf-8` and this meta element must be in the first 1kb of the document #} {% endblock %} {% block startHead %} @@ -15,7 +16,6 @@ {% block standardHead %} - {% endblock %} {% component '@apostrophecms/template:inject' with { where: 'head', end: 'append', when: 'hmr' } %} {% component '@apostrophecms/template:inject' with { where: 'head', end: 'append' } %} From d45e27f6d66cd25fde0cf9c7199ee9d014fa1518 Mon Sep 17 00:00:00 2001 From: haroun <1765606+haroun@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:10:15 +0200 Subject: [PATCH 05/56] Log aposResponse errors (#5386) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * log aposResponse errors * add changeset --------- Co-authored-by: Harouna Traoré --- .changeset/wild-lies-film.md | 5 +++++ packages/apostrophe-astro/lib/aposResponse.js | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 .changeset/wild-lies-film.md diff --git a/.changeset/wild-lies-film.md b/.changeset/wild-lies-film.md new file mode 100644 index 0000000000..4070e1aea6 --- /dev/null +++ b/.changeset/wild-lies-film.md @@ -0,0 +1,5 @@ +--- +"@apostrophecms/apostrophe-astro": minor +--- + +Log aposResponse errors diff --git a/packages/apostrophe-astro/lib/aposResponse.js b/packages/apostrophe-astro/lib/aposResponse.js index ea4dcfb509..1f9f676080 100644 --- a/packages/apostrophe-astro/lib/aposResponse.js +++ b/packages/apostrophe-astro/lib/aposResponse.js @@ -49,7 +49,7 @@ export default async function aposResponse(req) { const aposUrl = new URL(aposHost + pathname); aposUrl.search = url.search; - + // Prepare headers, excluding any specified in config const requestHeaders = {}; for (const [name, value] of req.headers) { @@ -111,6 +111,7 @@ export default async function aposResponse(req) { } // Skip unknown encodings silently } catch (decompressError) { + console.error(decompressError); // If decompression fails, return original response return new Response(new Uint8Array(bodyArrayBuffer), { ...rest, @@ -133,10 +134,12 @@ export default async function aposResponse(req) { headers: responseHeaders }); } catch (bodyError) { + console.error(bodyError); // If we can't process the body, fall back to the original response return new Response(res.body, { ...rest, status: statusCode, headers: responseHeaders }); } } catch (error) { + console.error(error); // Handle any unexpected errors return new Response(`Server error: ${error.message}`, { status: 500 }); } From f3501f4de0448d4d0f8a01a1db4dc2fcf52c6218 Mon Sep 17 00:00:00 2001 From: Tom Boutell Date: Thu, 23 Apr 2026 09:21:40 -0400 Subject: [PATCH 06/56] merge back the thanks (#5388) --- .changeset/wild-lies-film.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/wild-lies-film.md b/.changeset/wild-lies-film.md index 4070e1aea6..3b608070dd 100644 --- a/.changeset/wild-lies-film.md +++ b/.changeset/wild-lies-film.md @@ -2,4 +2,4 @@ "@apostrophecms/apostrophe-astro": minor --- -Log aposResponse errors +Log aposResponse errors server side in the Astro process. Thanks to [Harouna Traore](https://github.com/haroun). From 0d857713c88b2d9d6393600baa2ca9f75c2d97ab Mon Sep 17 00:00:00 2001 From: Tom Boutell Date: Thu, 23 Apr 2026 09:22:01 -0400 Subject: [PATCH 07/56] Postgres (#5365) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * postgres experimental WIP * astonishingly, all mocha tests of apostrophe pass with this * mocha tests pass, actual sites work * lint clean * listDatabases support, but changes are coming * wip * dump and restore updates * backpressure, adequate handling of ObjectId for our needs (becomes its hex representation) * mild performance optimization * profiling * testing issue resolved * refactored to db-connect module, introduced sqlite adapter * sqlite WIP * debugging * programmatic API for dump/restore/copy dbs * linting, documentation * MIT license * text ranking is more accurate, documentation is more complete * good full text search for sqlite * updates for compatibility with the rest of the public and private modules, plus a few fixes to genuinely ambiguous tests * requirements found by testing private modules * fixes from full cypress run * eslint passing * restore permissions * maximize atomicity * bug fixes * * exit properly when asset tests fail * "npm test" tests all three adapters * ignore claude-tools in eslint * postgres and sqlite-inclusive ci matrix attempt * clean up logs * We hit github's limit on total configurations because every package gets its own matrix. Solve that with grouping: * apostrophe core * All regular ecosystem packages other than core * non-database-requiring packges * mongodb-specific packages This will probably speed it up too because it won't have to spin up a container a bazillion times. * hardened the asset tests, made them less timing sensitive, fixed a bad commit resulting from the way they dodgily patch themselves without a robust cleanup mechanism * fix a root cause of asset test instability * log mess * implemented missing $size operator * test compatibility * advanced permission uses regex in $in * regex in $in * .db() should not make false promises in plain postgres mode, it should fail * ability to specify a default adapter * obsolete file * put escapeHost back where it belongs * dead code removal, test cleanup * emulate-mongo-3-driver only needed in db-connect * no claude logs in repo (tools are welcome) * * shared aggregation implementation, other shared things * optimize $match when it is the first step in aggregation, don't fetch the whole collection 😜 * multipostgres listDatabases() and .db() should return and expect "fully qualified virtual database names," e.g. physical_db_name-schemaname * vanilla postgres should not attempt to use .db() with alternate names in tests * documentation corrections * documentation errors * listDatabases and documentation corrections * more edge cases revealed by latest work from Miro * anchored prefix regexps are optimized documentation improvements * * matchesQuery in the aggregation cursor implementation doesn't throw on unrecognized operators. It should, and it should support the same mongodb operators that the regular find() path does in postgres/sqlite (our official subset), unless there is an extraordinary reason not to. * Similarly, the main query implementation for normal queries should throw on unrecognized operators if it doesn't already. * The dump/restore programmatic APIs in db-connect concern me. These involve returning the entire database as a string, which could exhaust memory. This impacts both utilities and also copyDatabase(). Could these APIs return and expect async iterators instead of strings? * The test "anchored regex on an indexed field uses a btree index search" runs explain on a query that's hardcoded in the test. Instead these SQL based adapters should expose a means to get the SQL for a query, so it can be directly tested. Otherwise this test proves nothing as changes to the adapter accumulate in future. * Why is this test searching for "at least 1" and not exactly 1? it('should find documents with null value', async function() { const docs = await db.collection('test').find({ value: null }).toArray(); // MongoDB matches both null and missing fields with { value: null } expect(docs.length).to.be.at.least(1); }); * What is the maximum size of a db-connect document in the postgres and sqlite adapters? * Update the copyright year in db-connect/LICENSE.md to 2025. * The db-connect README mentions: sqlite://:memory: What happens if you try to use .db('some-name') with that? I think it would be best to just not support throwaway in-memory sqlite databases because I doubt anyone would intentionally store a website in one. * do not swallow dump/restore errors on indexes * cover how to run the utilities * fix detection of source * separate sanitization for index names * regex prefix safety * pnpm --------- Co-authored-by: Thomas Boutell --- .github/workflows/monorepo.yml | 68 +- .../scripts/detect-impacted-packages.mjs | 7 +- .../scripts/expand-runtime-matrix.mjs | 126 +- .gitignore | 2 + claude-tools/run-core-tests.sh | 37 + package.json | 4 +- packages/apostrophe/.gitignore | 3 + .../apostrophe/claude-tools/detect-handles.js | 46 + .../claude-tools/minimal-hang-test.js | 28 + .../claude-tools/mongo-close-test.js | 11 + .../apostrophe/claude-tools/stdin-ref-test.js | 14 + packages/apostrophe/eslint.config.js | 3 +- .../modules/@apostrophecms/db/index.js | 95 +- .../modules/@apostrophecms/http/index.js | 2 +- .../modules/@apostrophecms/job/index.js | 16 +- packages/apostrophe/package.json | 8 +- .../apostrophe/scripts/find-heavy-npm-modules | 0 packages/apostrophe/test-lib/util.js | 64 +- .../add-missing-schema-fields-project/test.js | 14 +- packages/apostrophe/test/assets.js | 177 +- packages/apostrophe/test/db-tools.js | 365 ++ packages/apostrophe/test/db.js | 39 +- packages/apostrophe/test/default-adapter.js | 256 ++ packages/apostrophe/test/job.js | 2 +- packages/broadband/package.json | 3 + packages/cache-on-demand/package.json | 3 + packages/db-connect/LICENSE.md | 7 + packages/db-connect/README.md | 107 + packages/db-connect/adapters/mongodb.js | 10 + packages/db-connect/adapters/postgres.js | 2505 ++++++++++++ packages/db-connect/adapters/sqlite.js | 2398 ++++++++++++ packages/db-connect/bin/apos-db-dump.js | 42 + packages/db-connect/bin/apos-db-restore.js | 30 + .../claude-tools/logs/dump-mongodb.log | 11 + .../claude-tools/logs/dump-multipostgres.log | 11 + .../claude-tools/logs/dump-postgres.log | 11 + .../claude-tools/logs/dump-sqlite.log | 11 + packages/db-connect/docs/aggregation.md | 118 + packages/db-connect/docs/collections.md | 293 ++ packages/db-connect/docs/database.md | 94 + packages/db-connect/docs/dump-restore.md | 178 + packages/db-connect/docs/indexes.md | 114 + packages/db-connect/docs/queries.md | 233 ++ packages/db-connect/docs/updates.md | 99 + packages/db-connect/eslint.config.js | 6 + packages/db-connect/index.js | 44 + packages/db-connect/lib/aggregation-cursor.js | 408 ++ packages/db-connect/lib/copy-database.js | 11 + packages/db-connect/lib/dump.js | 91 + .../lib/mongodb-connect.js | 2 +- packages/db-connect/lib/restore.js | 166 + packages/db-connect/lib/shared.js | 438 +++ packages/db-connect/package.json | 29 + packages/db-connect/test/adapter.test.js | 3361 +++++++++++++++++ packages/db-connect/test/dump-restore.test.js | 180 + packages/db-connect/test/security.test.js | 316 ++ packages/db-connect/test/shared.test.js | 121 + packages/emulate-mongo-3-driver/package.json | 3 + packages/express-cache-on-demand/package.json | 3 + packages/form/test/test.js | 6 +- .../import-export/test/importDraftsOnly.js | 12 + packages/import-export/test/index.js | 93 +- .../import-export/test/overrideDuplicates.js | 2 +- .../import-export/test/overrideLocales.js | 5 + packages/mongodb-snapshot/package.json | 3 + packages/uploadfs/webp-test.js | 0 66 files changed, 12764 insertions(+), 201 deletions(-) create mode 100755 claude-tools/run-core-tests.sh create mode 100644 packages/apostrophe/claude-tools/detect-handles.js create mode 100644 packages/apostrophe/claude-tools/minimal-hang-test.js create mode 100644 packages/apostrophe/claude-tools/mongo-close-test.js create mode 100644 packages/apostrophe/claude-tools/stdin-ref-test.js mode change 100755 => 100644 packages/apostrophe/scripts/find-heavy-npm-modules create mode 100644 packages/apostrophe/test/db-tools.js create mode 100644 packages/apostrophe/test/default-adapter.js create mode 100644 packages/db-connect/LICENSE.md create mode 100644 packages/db-connect/README.md create mode 100644 packages/db-connect/adapters/mongodb.js create mode 100644 packages/db-connect/adapters/postgres.js create mode 100644 packages/db-connect/adapters/sqlite.js create mode 100755 packages/db-connect/bin/apos-db-dump.js create mode 100755 packages/db-connect/bin/apos-db-restore.js create mode 100644 packages/db-connect/claude-tools/logs/dump-mongodb.log create mode 100644 packages/db-connect/claude-tools/logs/dump-multipostgres.log create mode 100644 packages/db-connect/claude-tools/logs/dump-postgres.log create mode 100644 packages/db-connect/claude-tools/logs/dump-sqlite.log create mode 100644 packages/db-connect/docs/aggregation.md create mode 100644 packages/db-connect/docs/collections.md create mode 100644 packages/db-connect/docs/database.md create mode 100644 packages/db-connect/docs/dump-restore.md create mode 100644 packages/db-connect/docs/indexes.md create mode 100644 packages/db-connect/docs/queries.md create mode 100644 packages/db-connect/docs/updates.md create mode 100644 packages/db-connect/eslint.config.js create mode 100644 packages/db-connect/index.js create mode 100644 packages/db-connect/lib/aggregation-cursor.js create mode 100644 packages/db-connect/lib/copy-database.js create mode 100644 packages/db-connect/lib/dump.js rename packages/{apostrophe => db-connect}/lib/mongodb-connect.js (96%) create mode 100644 packages/db-connect/lib/restore.js create mode 100644 packages/db-connect/lib/shared.js create mode 100644 packages/db-connect/package.json create mode 100644 packages/db-connect/test/adapter.test.js create mode 100644 packages/db-connect/test/dump-restore.test.js create mode 100644 packages/db-connect/test/security.test.js create mode 100644 packages/db-connect/test/shared.test.js mode change 100755 => 100644 packages/uploadfs/webp-test.js diff --git a/.github/workflows/monorepo.yml b/.github/workflows/monorepo.yml index 84746c945f..4276a83fe0 100644 --- a/.github/workflows/monorepo.yml +++ b/.github/workflows/monorepo.yml @@ -7,6 +7,7 @@ env: NODE_VERSIONS_JSON: "[20,22,24]" MONGODB_VERSIONS_JSON: '["7","8"]' REDIS_VERSION: "7" + POSTGRES_VERSION: "16" on: push: @@ -124,7 +125,7 @@ jobs: run: | month='${{ steps.cache-month.outputs.value }}' mongo_suffix=$(jq -r '.[]' <<< '${{ env.MONGODB_VERSIONS_JSON }}' | paste -sd'-' -) - echo "key=docker-images-${month}-mongo${mongo_suffix}-redis${{ env.REDIS_VERSION }}" >> "$GITHUB_OUTPUT" + echo "key=docker-images-${month}-mongo${mongo_suffix}-redis${{ env.REDIS_VERSION }}-pg${{ env.POSTGRES_VERSION }}" >> "$GITHUB_OUTPUT" - name: Restore docker image cache id: docker-cache @@ -143,6 +144,8 @@ jobs: done docker pull "redis:${{ env.REDIS_VERSION }}" docker save "redis:${{ env.REDIS_VERSION }}" -o ".github/docker-cache/redis-${{ env.REDIS_VERSION }}.tar" + docker pull "postgres:${{ env.POSTGRES_VERSION }}" + docker save "postgres:${{ env.POSTGRES_VERSION }}" -o ".github/docker-cache/postgres-${{ env.POSTGRES_VERSION }}.tar" warm-sharp-cache: name: Warm sharp/libvips cache (Node ${{ matrix.nodeVersion }}) @@ -216,7 +219,7 @@ jobs: du -sh ~/.npm/_libvips || true package-tests: - name: ${{ format('{0} ({1}, {2})', matrix.package, matrix.nodeVersion, matrix.needsMongo && matrix.mongodbVersion || 'n/a') }} + name: ${{ format('{0} ({1}, {2}{3})', matrix.group, matrix.nodeVersion, matrix.adapter || 'n/a', matrix.needsMongo && format(' mongo {0}', matrix.mongodbVersion) || '') }} needs: - setup - shared-runtime @@ -279,7 +282,7 @@ jobs: - name: Restore docker image cache id: docker-cache-restore uses: actions/cache/restore@v4 - if: matrix.needsMongo || matrix.needsRedis + if: matrix.needsMongo || matrix.needsRedis || matrix.needsPostgres with: path: .github/docker-cache key: ${{ needs.shared-runtime.outputs.docker-cache-key }} @@ -300,6 +303,14 @@ jobs: if: steps.docker-cache-restore.outputs.cache-hit != 'true' && matrix.needsRedis run: docker pull redis:${{ env.REDIS_VERSION }} + - name: Load cached Postgres image + if: steps.docker-cache-restore.outputs.cache-hit == 'true' && matrix.needsPostgres + run: docker load -i ".github/docker-cache/postgres-${{ env.POSTGRES_VERSION }}.tar" + + - name: Pull Postgres image (cache miss) + if: steps.docker-cache-restore.outputs.cache-hit != 'true' && matrix.needsPostgres + run: docker pull postgres:${{ env.POSTGRES_VERSION }} + - name: Start MongoDB if: matrix.needsMongo run: | @@ -358,15 +369,64 @@ jobs: docker logs redis exit 1 + - name: Start PostgreSQL + if: matrix.needsPostgres + run: | + docker rm -f postgres >/dev/null 2>&1 || true + docker run -d \ + --name postgres \ + --publish 5432:5432 \ + -e POSTGRES_HOST_AUTH_METHOD=trust \ + --health-cmd "pg_isready -U postgres" \ + --health-interval 5s \ + --health-timeout 5s \ + --health-retries 12 \ + postgres:${{ env.POSTGRES_VERSION }} + echo "Waiting for PostgreSQL to report healthy..." + for attempt in $(seq 1 60); do + status=$(docker inspect --format='{{.State.Health.Status}}' postgres 2>/dev/null || echo "starting") + if [ "$status" = "healthy" ]; then + exit 0 + fi + if [ "$status" = "unhealthy" ]; then + echo "PostgreSQL reported unhealthy" >&2 + docker logs postgres + exit 1 + fi + sleep 2 + done + echo "PostgreSQL failed to become healthy in time" >&2 + docker logs postgres + exit 1 + - name: Install workspace dependencies run: pnpm install --frozen-lockfile - name: Run package tests - run: pnpm run --filter "${{ matrix.package }}" --if-present test + run: | + failed=0 + for pkg in $(echo '${{ matrix.packages }}' | jq -r '.[]'); do + echo "::group::Testing $pkg" + if ! pnpm run --filter "$pkg" --if-present test; then + echo "::error::Tests failed for $pkg" + failed=1 + fi + echo "::endgroup::" + done + exit $failed env: CI: true # Yes we want import-export to test with automatic-translation TEST_WITH_PRO: "1" + # Adapter selection: mongodb (default), postgres, or sqlite. + # ADAPTER is used by db-connect tests, APOS_TEST_DB_PROTOCOL by apostrophe tests. + ADAPTER: ${{ matrix.adapter }} + APOS_TEST_DB_PROTOCOL: ${{ matrix.adapter }} + PGUSER: postgres + + - name: Stop PostgreSQL + if: always() && matrix.needsPostgres + run: docker rm -f postgres || true - name: Stop Redis if: always() && matrix.needsRedis diff --git a/.github/workflows/scripts/detect-impacted-packages.mjs b/.github/workflows/scripts/detect-impacted-packages.mjs index 453f370938..392a29b6d1 100644 --- a/.github/workflows/scripts/detect-impacted-packages.mjs +++ b/.github/workflows/scripts/detect-impacted-packages.mjs @@ -73,7 +73,8 @@ async function main() { package: name, directory: packages.get(name).relativeDir, requiresMongo: packages.get(name).requiresMongo !== false, - requiresRedis: packages.get(name).requiresRedis === true + requiresRedis: packages.get(name).requiresRedis === true, + mongodbOnly: packages.get(name).mongodbOnly === true })) }; @@ -130,6 +131,7 @@ async function loadPackages() { const testConfig = manifest.apostropheTestConfig || {}; const requiresMongo = testConfig.requiresMongo !== false; const requiresRedis = testConfig.requiresRedis === true; + const mongodbOnly = testConfig.mongodbOnly === true; map.set(manifest.name, { name: manifest.name, @@ -137,7 +139,8 @@ async function loadPackages() { dependencies, hasTestScript, requiresMongo, - requiresRedis + requiresRedis, + mongodbOnly }); })); diff --git a/.github/workflows/scripts/expand-runtime-matrix.mjs b/.github/workflows/scripts/expand-runtime-matrix.mjs index 0c0e46f690..3672fad06b 100644 --- a/.github/workflows/scripts/expand-runtime-matrix.mjs +++ b/.github/workflows/scripts/expand-runtime-matrix.mjs @@ -1,6 +1,21 @@ #!/usr/bin/env node -// Expands the impacted package matrix with runtime permutations supplied +// Expands the impacted package matrix with runtime permutations supplied // via env vars. +// +// Packages are grouped into jobs to stay within GitHub's 256-entry matrix limit: +// - "apostrophe" runs solo (the main package, benefits from its own status). +// - All other database packages are grouped into an "ecosystem" job. +// - mongodbOnly packages are grouped into "ecosystem-mongodb". +// - Non-database packages are grouped into "standalone". +// +// For groups that need a database, three adapter variants are emitted: +// 1. mongodb – all Node versions × all MongoDB versions +// 2. postgres – latest LTS Node only, no MongoDB +// 3. sqlite – latest LTS Node only, no MongoDB +// +// The latest LTS Node version is the highest even-numbered entry in +// NODE_VERSIONS_JSON. + import { readFile } from 'fs/promises'; const args = process.argv.slice(2); @@ -36,25 +51,120 @@ function parseJsonArray(name, raw) { const nodeVersions = parseJsonArray('NODE_VERSIONS_JSON', process.env.NODE_VERSIONS_JSON); const mongodbVersions = parseJsonArray('MONGODB_VERSIONS_JSON', process.env.MONGODB_VERSIONS_JSON); +// Latest LTS = highest even-numbered Node version +const latestLts = [...nodeVersions] + .filter((v) => Number(v) % 2 === 0) + .sort((a, b) => Number(b) - Number(a))[0]; + const impact = JSON.parse(await readFile(impactPath, 'utf8')); const packages = impact?.matrix?.include || []; -const include = []; + +// The main apostrophe package always gets its own jobs for clear CI status. +const SOLO_PACKAGES = new Set(['apostrophe']); + +// Sort packages into groups +const solo = []; +const ecosystem = []; +const ecosystemMongodbOnly = []; +const standalone = []; for (const pkg of packages) { - const needsMongo = pkg.requiresMongo !== false; - const needsRedis = pkg.requiresRedis === true; - const mongoTargets = needsMongo ? mongodbVersions : ['']; + const needsDb = pkg.requiresMongo !== false; + if (SOLO_PACKAGES.has(pkg.package)) { + solo.push(pkg); + } else if (needsDb && pkg.mongodbOnly) { + ecosystemMongodbOnly.push(pkg); + } else if (needsDb) { + ecosystem.push(pkg); + } else { + standalone.push(pkg); + } +} + +const include = []; + +// Emit runtime combinations for a group of packages. +function emitGroup(group, pkgs) { + if (!pkgs.length) { + return; + } + const packageNames = JSON.stringify(pkgs.map((p) => p.package)); + const needsRedis = pkgs.some((p) => p.requiresRedis === true); + const mongodbOnly = pkgs.every((p) => p.mongodbOnly); + + // mongodb: all Node versions × all MongoDB versions for (const nodeVersion of nodeVersions) { - for (const mongodbVersion of mongoTargets) { + for (const mongodbVersion of mongodbVersions) { include.push({ - ...pkg, + group, + packages: packageNames, nodeVersion, mongodbVersion, - needsMongo, + adapter: 'mongodb', + needsMongo: true, + needsPostgres: false, needsRedis }); } } + // postgres and sqlite: latest LTS only, skip for mongodb-only groups + if (!mongodbOnly) { + include.push({ + group, + packages: packageNames, + nodeVersion: latestLts, + mongodbVersion: '', + adapter: 'postgres', + needsMongo: false, + needsPostgres: true, + needsRedis + }); + include.push({ + group, + packages: packageNames, + nodeVersion: latestLts, + mongodbVersion: '', + adapter: 'sqlite', + needsMongo: false, + needsPostgres: false, + needsRedis + }); + } +} + +// Emit non-database group (no adapter variants, just Node versions) +function emitStandalone(group, pkgs) { + if (!pkgs.length) { + return; + } + const packageNames = JSON.stringify(pkgs.map((p) => p.package)); + for (const nodeVersion of nodeVersions) { + include.push({ + group, + packages: packageNames, + nodeVersion, + mongodbVersion: '', + adapter: '', + needsMongo: false, + needsPostgres: false, + needsRedis: false + }); + } +} + +// Solo packages each get their own group +for (const pkg of solo) { + emitGroup(pkg.package, [pkg]); +} + +if (ecosystem.length) { + emitGroup('ecosystem', ecosystem); +} +if (ecosystemMongodbOnly.length) { + emitGroup('ecosystem-mongodb', ecosystemMongodbOnly); +} +if (standalone.length) { + emitStandalone('standalone', standalone); } process.stdout.write(JSON.stringify({ include })); diff --git a/.gitignore b/.gitignore index 6bccc37b4b..e88b2189a4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ public/apos-frontend .DS_Store coverage/ .nyc_output +claude-tools/logs/ +.claude diff --git a/claude-tools/run-core-tests.sh b/claude-tools/run-core-tests.sh new file mode 100755 index 0000000000..eb8b77c009 --- /dev/null +++ b/claude-tools/run-core-tests.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Run the apostrophe core test suite against a chosen DB adapter and log +# output to claude-tools/logs/core-.log. Usage: +# +# ./claude-tools/run-core-tests.sh mongodb +# ./claude-tools/run-core-tests.sh postgres +# ./claude-tools/run-core-tests.sh sqlite +# +# NEVER run multiple adapters in parallel — the test suite is not designed +# for concurrent runs and the host has limited resources. + +set -u +adapter="${1:-}" +if [[ -z "$adapter" ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +root="$(cd "$(dirname "$0")/.." && pwd)" +logdir="$root/claude-tools/logs" +mkdir -p "$logdir" +log="$logdir/core-$adapter.log" +: > "$log" + +echo "=== $adapter core tests ($(date -Is)) ===" | tee -a "$log" + +cd "$root/packages/apostrophe" + +extra=() +if [[ "$adapter" == "postgres" ]]; then + extra=(env PGPASSWORD=testpassword) +fi + +APOS_TEST_DB_PROTOCOL="$adapter" "${extra[@]}" npm run test:base >> "$log" 2>&1 +code=$? +echo "=== exit=$code ===" | tee -a "$log" +exit "$code" diff --git a/package.json b/package.json index 1fd8ccdf94..c80aee125e 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "dev": "pnpm --parallel --recursive run dev", "build": "pnpm --recursive run build", "lint": "pnpm --recursive run lint", - "test": "pnpm --recursive run test", + "test": "APOS_TEST_DB_PROTOCOL=postgres pnpm run test:main && APOS_TEST_DB_PROTOCOL=mongodb pnpm run test:main && APOS_TEST_DB_PROTOCOL=sqlite pnpm run test:main && APOS_TEST_DB_PROTOCOL=multipostgres pnpm run test:main", + "test:main": "echo \"APOS_TEST_DB_PROTOCOL IS: $APOS_TEST_DB_PROTOCOL\" && pnpm --recursive run test", "eslint": "pnpm --recursive run eslint", "mocha": "pnpm --recursive run mocha", "clean": "pnpm -r exec rm -rf node_modules && rm -rf node_modules && rm pnpm-lock.yaml" @@ -18,6 +19,7 @@ }, "pnpm": { "onlyBuiltDependencies": [ + "better-sqlite3", "sharp", "vue-demi", "@parcel/watcher", diff --git a/packages/apostrophe/.gitignore b/packages/apostrophe/.gitignore index 894a9d6ba3..16ad414a99 100644 --- a/packages/apostrophe/.gitignore +++ b/packages/apostrophe/.gitignore @@ -42,3 +42,6 @@ test/public/uploads # vim swp files .*.sw* + +# claude-tools log files +claude-tools/**/*.log diff --git a/packages/apostrophe/claude-tools/detect-handles.js b/packages/apostrophe/claude-tools/detect-handles.js new file mode 100644 index 0000000000..9cf711d976 --- /dev/null +++ b/packages/apostrophe/claude-tools/detect-handles.js @@ -0,0 +1,46 @@ +// Require this before running mocha to detect what activates process.stdin +// Usage: npx mocha -t 10000 --require ./claude-tools/detect-handles.js test/assets.js + +console.log(`stdin paused at startup: ${process.stdin.isPaused()}`); +console.log(`stdin readableFlowing at startup: ${process.stdin.readableFlowing}`); + +// Monkey-patch stdin.resume to capture the call stack +const origResume = process.stdin.resume.bind(process.stdin); +process.stdin.resume = function(...args) { + console.log('\n=== process.stdin.resume() called ==='); + console.log(new Error().stack); + return origResume(...args); +}; + +// Monkey-patch stdin.on to detect 'data' listener additions +const origOn = process.stdin.on.bind(process.stdin); +process.stdin.on = function(event, ...args) { + if (event === 'data' || event === 'readable') { + console.log(`\n=== process.stdin.on('${event}') called ===`); + console.log(new Error().stack); + } + return origOn(event, ...args); +}; + +// Periodically check stdin state changes +let lastState = process.stdin.readableFlowing; +const checker = setInterval(() => { + if (process.stdin.readableFlowing !== lastState) { + console.log(`\n=== stdin readableFlowing changed: ${lastState} -> ${process.stdin.readableFlowing} ===`); + console.log(new Error().stack); + lastState = process.stdin.readableFlowing; + } +}, 100); +checker.unref(); + +const origRun = require('mocha/lib/runner').prototype.run; +require('mocha/lib/runner').prototype.run = function(fn) { + return origRun.call(this, function(failures) { + console.log(`\nstdin paused at end: ${process.stdin.isPaused()}`); + console.log(`stdin readableFlowing at end: ${process.stdin.readableFlowing}`); + setTimeout(() => { + process.exit(failures ? 3 : 0); + }, 2000); + if (fn) fn(failures); + }); +}; diff --git a/packages/apostrophe/claude-tools/minimal-hang-test.js b/packages/apostrophe/claude-tools/minimal-hang-test.js new file mode 100644 index 0000000000..3d29952b7f --- /dev/null +++ b/packages/apostrophe/claude-tools/minimal-hang-test.js @@ -0,0 +1,28 @@ +// Minimal test to isolate what causes the hang. +// Must reference the test/ directory as root for proper module resolution. +const t = require('../test-lib/test.js'); +const path = require('path'); + +// Fake a module object rooted in test/ like the real tests do +const fakeModule = { + id: path.join(__dirname, '../test/fake'), + filename: path.join(__dirname, '../test/fake.js'), + paths: [path.join(__dirname, '../test/node_modules')] +}; + +describe('Minimal hang test', function() { + this.timeout(60000); + let apos; + + after(async function() { + await t.destroy(apos); + console.log('after: destroy complete'); + }); + + it('should create and use apos without hanging', async function() { + apos = await t.create({ + root: fakeModule + }); + console.log('apos created successfully'); + }); +}); diff --git a/packages/apostrophe/claude-tools/mongo-close-test.js b/packages/apostrophe/claude-tools/mongo-close-test.js new file mode 100644 index 0000000000..360ce44299 --- /dev/null +++ b/packages/apostrophe/claude-tools/mongo-close-test.js @@ -0,0 +1,11 @@ +// Test whether a MongoDB connection keeps the process alive after close() +const mongoConnect = require('../../../packages/db-connect/lib/mongodb-connect'); + +(async () => { + const uri = 'mongodb://localhost:27017/test_handle_leak'; + console.log('Connecting...'); + const client = await mongoConnect(uri); + console.log('Connected. Closing...'); + await client.close(); + console.log('Closed. Process should exit now if no leaked handles.'); +})(); diff --git a/packages/apostrophe/claude-tools/stdin-ref-test.js b/packages/apostrophe/claude-tools/stdin-ref-test.js new file mode 100644 index 0000000000..d562211bcb --- /dev/null +++ b/packages/apostrophe/claude-tools/stdin-ref-test.js @@ -0,0 +1,14 @@ +// Check if process.stdin keeps the process alive +// If this script hangs, stdin is ref'd. If it exits, stdin is unref'd. + +console.log(`stdin isTTY: ${process.stdin.isTTY}`); +console.log(`stdin readableFlowing: ${process.stdin.readableFlowing}`); +console.log(`stdin isPaused: ${process.stdin.isPaused()}`); + +// Check ref status +if (typeof process.stdin.unref === 'function') { + console.log('stdin has unref method'); +} + +console.log('Waiting to see if process exits on its own...'); +// Don't do anything - just see if the process exits diff --git a/packages/apostrophe/eslint.config.js b/packages/apostrophe/eslint.config.js index 9377dc1974..969698e6ef 100644 --- a/packages/apostrophe/eslint.config.js +++ b/packages/apostrophe/eslint.config.js @@ -7,7 +7,8 @@ module.exports = defineConfig([ '**/blueimp/**/*.js', 'test/public', 'test/apos-build', - 'coverage' + 'coverage', + 'claude-tools' ]), apostrophe ]); diff --git a/packages/apostrophe/modules/@apostrophecms/db/index.js b/packages/apostrophe/modules/@apostrophecms/db/index.js index 8894adba94..58fac171ff 100644 --- a/packages/apostrophe/modules/@apostrophecms/db/index.js +++ b/packages/apostrophe/modules/@apostrophecms/db/index.js @@ -4,11 +4,12 @@ // // ### `uri` // -// The MongoDB connection URI. See the [MongoDB URI documentation](https://docs.mongodb.com/manual/reference/connection-string/). +// The databse connection URI. See the [MongoDB URI documentation](https://docs.mongodb.com/manual/reference/connection-string/) +// and the postgres documentation. // // ### `connect` // -// If present, this object is passed on as options to MongoDB's "connect" +// If present, this object is passed on as options to the database adapters "connect" // method, along with the uri. See the [MongoDB connect settings documentation](http://mongodb.github.io/node-mongodb-native/2.2/reference/connecting/connection-settings/). // // By default, Apostrophe sets options to retry lost connections forever, @@ -20,9 +21,16 @@ // // ### `client` // -// An existing MongoDB connection (MongoClient) object. If present, it is used +// An existing MongoDB-compatible client object. If present, it is used // and `uri`, `host`, `connect`, etc. are ignored. // +// ### `adapters` +// +// An array of adapters, each of which must provide `name`, `connect(uri, options)`, +// and `protocols` properties. `name` may be used to override a core adapter, +// such as `postgres` or `mongodb`. `connect` must resolve to a client object +// supporting a sufficient subset of the mongodb API. +// // ### `versionCheck` // // If `true`, check to make sure the database does not belong to an @@ -49,15 +57,15 @@ // in your project. However you may find it easier to just use the // `client` option. -const mongodbConnect = require('../../../lib/mongodb-connect'); -const escapeHost = require('../../../lib/escape-host'); +const dbConnect = require('@apostrophecms/db-connect'); +const escapeHost = require('../../../lib/escape-host.js'); module.exports = { options: { versionCheck: true }, async init(self) { - await self.connectToMongo(); + await self.connectToDb(); await self.versionCheck(); }, handlers(self) { @@ -81,14 +89,12 @@ module.exports = { }, methods(self) { return { - // Open the database connection. Always uses MongoClient with its - // sensible defaults. Builds a URI if necessary, so we can call it - // in a consistent way. - // - // One default we override: if the connection is lost, we keep - // attempting to reconnect forever. This is the most sensible behavior - // for a persistent process that requires MongoDB in order to operate. - async connectToMongo() { + // Connect to the database and sets self.apos.dbClient + // and self.apos.db. Builds a mongodb URI by default, + // accepting host, port, user, password and name options + // if present. More typically a URI is specified via + // APOS_DB_URI, or via APOS_MONGODB_URI for bc. + async connectToDb() { if (self.options.client) { // Reuse a single client connection http://mongodb.github.io/node-mongodb-native/2.2/api/Db.html#db self.apos.dbClient = self.options.client; @@ -96,32 +102,67 @@ module.exports = { self.connectionReused = true; return; } - let uri = 'mongodb://'; - if (process.env.APOS_MONGODB_URI) { - uri = process.env.APOS_MONGODB_URI; + let uri; + const viaEnv = process.env.APOS_DB_URI || process.env.APOS_MONGODB_URI; + if (viaEnv) { + uri = viaEnv; } else if (self.options.uri) { uri = self.options.uri; } else { - if (self.options.user) { - uri += self.options.user + ':' + self.options.password + '@'; - } - if (!self.options.host) { - self.options.host = 'localhost'; - } - if (!self.options.port) { - self.options.port = 27017; + const validAdapters = [ 'mongodb', 'sqlite', 'postgres', 'multipostgres' ]; + const adapter = process.env.APOS_DEFAULT_DB_ADAPTER || self.options.defaultAdapter || 'mongodb'; + if (!validAdapters.includes(adapter)) { + throw new Error(`Invalid defaultAdapter: "${adapter}". Must be one of: ${validAdapters.join(', ')}`); } if (!self.options.name) { self.options.name = self.apos.shortName; } - uri += escapeHost(self.options.host) + ':' + self.options.port + '/' + self.options.name; + if (adapter === 'sqlite') { + const path = require('path'); + uri = `sqlite://${path.resolve(self.apos.rootDir, 'data', self.options.name + '.sqlite')}`; + } else { + const credentials = self.options.user + ? encodeURIComponent(self.options.user) + ':' + encodeURIComponent(self.options.password) + '@' + : ''; + if (adapter === 'mongodb') { + if (!self.options.host) { + self.options.host = 'localhost'; + } + if (!self.options.port) { + self.options.port = 27017; + } + uri = 'mongodb://' + credentials + escapeHost(self.options.host) + ':' + self.options.port + '/' + self.options.name; + } else { + // postgres or multipostgres + if (!self.options.host) { + self.options.host = 'localhost'; + } + if (!self.options.port) { + self.options.port = 5432; + } + uri = adapter + '://' + credentials + escapeHost(self.options.host) + ':' + self.options.port + '/' + self.options.name; + } + } } - self.apos.dbClient = await mongodbConnect(uri, self.options.connect); + self.apos.dbClient = await dbConnect(uri, { + ...self.options.connect, + adapters: self.options.adapters + }); self.uri = uri; // Automatically uses the db name in the connection string self.apos.db = self.apos.dbClient.db(); }, + // Connect to a database using the appropriate adapter based on the URI protocol. + // Returns a client object compatible with the MongoDB driver interface. + // This method has no side effects — it does not set apos.db or apos.dbClient. + // It can be used to make temporary connections, e.g. for dropping a test database. + async connectToAdapter(uri, options) { + return dbConnect(uri, { + ...options, + adapters: self.options.adapters + }); + }, async versionCheck() { if (!self.options.versionCheck) { return; diff --git a/packages/apostrophe/modules/@apostrophecms/http/index.js b/packages/apostrophe/modules/@apostrophecms/http/index.js index f7a7b04797..a715259ca3 100644 --- a/packages/apostrophe/modules/@apostrophecms/http/index.js +++ b/packages/apostrophe/modules/@apostrophecms/http/index.js @@ -2,7 +2,7 @@ const _ = require('lodash'); const qs = require('qs'); const fetch = require('node-fetch'); const tough = require('tough-cookie'); -const escapeHost = require('../../../lib/escape-host'); +const escapeHost = require('../../../lib/escape-host.js'); const util = require('util'); module.exports = { diff --git a/packages/apostrophe/modules/@apostrophecms/job/index.js b/packages/apostrophe/modules/@apostrophecms/job/index.js index eeda4a2f14..425e5a85d4 100644 --- a/packages/apostrophe/modules/@apostrophecms/job/index.js +++ b/packages/apostrophe/modules/@apostrophecms/job/index.js @@ -244,7 +244,9 @@ module.exports = { }, setTotal (n) { total = n; - return self.setTotal(job, n); + const result = self.setTotal(job, n); + promises.push(result); + return result; }, setResults (_results) { results = _results; @@ -412,12 +414,12 @@ module.exports = { // // No promise is returned as this method just updates // the job tracking information in the background. - setTotal(job, total) { - self.db.updateOne({ _id: job._id }, { $set: { total } }, function (err) { - if (err) { - self.apos.util.error(err); - } - }); + async setTotal(job, total) { + try { + await self.db.updateOne({ _id: job._id }, { $set: { total } }); + } catch (err) { + self.apos.util.error(err); + } }, // Mark the given job as ended. If `success` // is true the job is reported as an overall diff --git a/packages/apostrophe/package.json b/packages/apostrophe/package.json index 09b882f9bf..a01079b6d2 100644 --- a/packages/apostrophe/package.json +++ b/packages/apostrophe/package.json @@ -5,10 +5,9 @@ "main": "index.js", "scripts": { "pretest": "npm run lint", - "test": "npm run test:base && npm run test:missing && npm run test:assets && npm run test:esm", - "test:base": "nyc mocha -t 10000 --ignore=test/assets.js", + "test": "npm run test:base && npm run test:missing && npm run test:esm", + "test:base": "nyc mocha -t 10000", "test:missing": "nyc mocha -t 10000 test/add-missing-schema-fields-project/test.js", - "test:assets": "nyc mocha -t 10000 test/assets.js", "test:esm": "mocha -t 1000 test/esm-project/esm.js", "eslint": "eslint .", "eslint-fix": "npm run eslint -- --fix", @@ -38,7 +37,7 @@ "author": "Apostrophe Technologies, Inc.", "license": "MIT", "dependencies": { - "@apostrophecms/emulate-mongo-3-driver": "workspace:^", + "@apostrophecms/db-connect": "workspace:^", "@apostrophecms/vue-material-design-icons": "^1.0.0", "@ctrl/tinycolor": "^4.1.0", "@floating-ui/dom": "^1.5.3", @@ -141,6 +140,7 @@ "xregexp": "^2.0.0" }, "devDependencies": { + "chai": "^4.3.10", "eslint": "^9.39.1", "eslint-config-apostrophe": "workspace:^", "form-data": "^4.0.4", diff --git a/packages/apostrophe/scripts/find-heavy-npm-modules b/packages/apostrophe/scripts/find-heavy-npm-modules old mode 100755 new mode 100644 diff --git a/packages/apostrophe/test-lib/util.js b/packages/apostrophe/test-lib/util.js index 2f60d88f8f..373d8bfd9b 100644 --- a/packages/apostrophe/test-lib/util.js +++ b/packages/apostrophe/test-lib/util.js @@ -1,5 +1,27 @@ const { createId } = require('@paralleldrive/cuid2'); -const mongodbConnect = require('../lib/mongodb-connect'); + +const testDbProtocol = process.env.APOS_TEST_DB_PROTOCOL || 'mongodb'; + +// Build a test database URI for postgres based on the shortName. +// Returns undefined for mongodb, letting the default logic handle it. +function getTestDbUri(shortName) { + if (testDbProtocol === 'postgres') { + // PostgreSQL database names cannot contain hyphens + const dbName = shortName.replace(/-/g, '_'); + return `postgres://localhost:5432/${dbName}`; + } + if (testDbProtocol === 'multipostgres') { + // Multi-schema mode: shared real database, per-test schema + const schemaName = shortName.replace(/-/g, '_').replace(/[^a-zA-Z0-9_]/g, ''); + return `multipostgres://localhost:5432/apos_test-${schemaName}`; + } + if (testDbProtocol === 'sqlite') { + const os = require('os'); + const path = require('path'); + const dbName = shortName.replace(/-/g, '_').replace(/[^a-zA-Z0-9_]/g, ''); + return `sqlite://${path.join(os.tmpdir(), `apos_test_${dbName}.db`)}`; + } +} // Properly clean up an apostrophe instance and drop its // database collections to create a sane environment for the next test. @@ -10,23 +32,23 @@ const mongodbConnect = require('../lib/mongodb-connect'); // If `apos` is null, no work is done. async function destroy(apos) { - if (!apos) { + if (!apos || apos._destroyed) { return; } + apos._destroyed = true; + const dbModule = apos.modules['@apostrophecms/db']; + const { uri } = dbModule; + const dbName = apos.db && (apos.db.databaseName || apos.db._name); await apos.destroy(); - const { uri } = apos.modules['@apostrophecms/db']; - const dbName = apos.db && apos.db.databaseName; - // TODO at some point accommodate nonsense like testing remote databases - // that won't let us use dropDatabase, no shell available etc., but the - // important principle here is that we should not have to have an apos - // object to clean up the database, otherwise we have to get hold of one - // when initialization failed and that's really not apostrophe's concern - if (dbName && uri) { - const client = await mongodbConnect(`${uri}${dbName}`); - const db = client.db(dbName); - await db.dropDatabase(); - await client.close(); + if (!uri || !dbName) { + return; } + // Make a fresh connection (the original was closed by destroy) + // and use it to drop the test database + const client = await dbModule.connectToAdapter(uri); + const db = client.db(dbName); + await db.dropDatabase(); + await client.close(); }; async function create(options = {}) { @@ -55,6 +77,18 @@ async function create(options = {}) { express.options.session.secret = express.options.session.secret || 'test'; config.modules['@apostrophecms/express'] = express; } + // When APOS_TEST_DB_PROTOCOL=postgres, automatically configure the db + // module to use a postgres URI unless already explicitly configured + const testUri = getTestDbUri(config.shortName); + if (testUri) { + config.modules = config.modules || {}; + const dbModule = config.modules['@apostrophecms/db'] || {}; + dbModule.options = dbModule.options || {}; + if (!dbModule.options.uri && !dbModule.options.client) { + dbModule.options.uri = testUri; + } + config.modules['@apostrophecms/db'] = dbModule; + } return require('../index.js')(config); } @@ -151,5 +185,7 @@ module.exports = { loginAs, logout, getUserJar, + getTestDbUri, + testDbProtocol, timeout: (process.env.TEST_TIMEOUT && parseInt(process.env.TEST_TIMEOUT)) || 20000 }; diff --git a/packages/apostrophe/test/add-missing-schema-fields-project/test.js b/packages/apostrophe/test/add-missing-schema-fields-project/test.js index f2c142c3be..819a96c445 100644 --- a/packages/apostrophe/test/add-missing-schema-fields-project/test.js +++ b/packages/apostrophe/test/add-missing-schema-fields-project/test.js @@ -11,10 +11,18 @@ describe('Apostrophe - add-missing-schema-fields task', function() { let apos; + // When APOS_TEST_DB_PROTOCOL is set, child processes that run `node app.js` + // need the matching APOS_DB_URI so they use the same database as t.create() + const projectCwd = path.resolve(process.cwd(), 'test/add-missing-schema-fields-project/'); + const testDbUri = t.getTestDbUri('add-missing-schema-fields-project'); + const execEnv = testDbUri + ? { env: { ...process.env, APOS_DB_URI: testDbUri } } + : {}; + before(async function() { await util.promisify(exec)( 'npm install', - { cwd: path.resolve(process.cwd(), 'test/add-missing-schema-fields-project/') } + { cwd: projectCwd } ); }); @@ -25,7 +33,7 @@ describe('Apostrophe - add-missing-schema-fields task', function() { it('should not run migrations when running the task', async function() { await util.promisify(exec)( 'node app.js @apostrophecms/migration:add-missing-schema-fields', - { cwd: path.resolve(process.cwd(), 'test/add-missing-schema-fields-project/') } + { cwd: projectCwd, ...execEnv } ); apos = await t.create({ @@ -64,7 +72,7 @@ describe('Apostrophe - add-missing-schema-fields task', function() { it('should run migrations when running @apostrophecms/migration:migrate task', async function() { await util.promisify(exec)( 'node app.js @apostrophecms/migration:migrate', - { cwd: path.resolve(process.cwd(), 'test/add-missing-schema-fields-project/') } + { cwd: projectCwd, ...execEnv } ); apos = await t.create({ diff --git a/packages/apostrophe/test/assets.js b/packages/apostrophe/test/assets.js index f5b31454b2..afe6538656 100644 --- a/packages/apostrophe/test/assets.js +++ b/packages/apostrophe/test/assets.js @@ -98,15 +98,75 @@ describe('Assets', function() { retryAssertTrue } = loadUtils(); + // Wait for the chokidar watcher to be ready before writing files. + // Without this, writes can happen before chokidar has finished its + // initial scan, so the change event is never emitted. This is + // especially visible with slower adapters like sqlite. + function waitForWatcherReady(watcher, timeoutMs = 10000) { + if (watcher._readyEmitted) { + return Promise.resolve(); + } + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('Watcher ready timeout')), timeoutMs); + watcher.on('ready', () => { + clearTimeout(timer); + resolve(); + }); + }); + } + + // Many asset tests modify source files in test/modules/ to trigger + // rebuilds, then restore them at the end. If a test fails mid-execution + // the files stay dirty and poison subsequent runs. To prevent this we + // snapshot every mutable file before the suite and restore them + // automatically after each test via afterEach. The before hook also + // cleans up build artifacts and webpack cache from prior runs. + const mutableFiles = [ + 'test/modules/bundle-page/ui/src/extra.js', + 'test/modules/default-page/ui/src/index.js', + 'test/modules/default-page/ui/src/index.scss', + 'test/modules/default-page/ui/public/index.js', + 'test/modules/default-page/ui/public/index.css', + 'test/modules/default-page/ui/apos/components/FakeComponent.vue', + 'test/package-lock.json' + ].map((rel) => path.join(process.cwd(), rel)); + const snapshots = new Map(); + + before(async function() { + // Snapshot every mutable file so afterEach can restore them + for (const file of mutableFiles) { + try { + snapshots.set(file, await fs.readFile(file)); + } catch (e) { + // File might not exist yet, that's OK + } + } + // Start clean: remove build artifacts and cache from prior runs + await deleteBuiltFolders(publicFolderPath, true); + await removeCache(); + }); + after(async function() { await deleteBuiltFolders(publicFolderPath, true); await removeCache(); await t.destroy(apos); }); - afterEach(function() { + afterEach(async function() { // Prevent hang forever if particular tests fail while testing prod. process.env.NODE_ENV = 'development'; + // Restore any files that were modified by the test + for (const [ file, content ] of snapshots) { + try { + const current = await fs.readFile(file); + if (!current.equals(content)) { + await fs.writeFile(file, content); + } + } catch (e) { + // If the file was deleted, restore it + await fs.writeFile(file, content); + } + } }); this.timeout(5 * 60 * 1000); @@ -162,6 +222,7 @@ describe('Assets', function() { }); it('should get webpack extensions from modules and fill extra bundles', async function () { + await t.destroy(apos); const expectedEntryPointsNames = { js: [ 'company', 'main', 'another', 'extra', 'extra2' ], css: [ 'company', 'main', 'extra' ] @@ -309,6 +370,7 @@ describe('Assets', function() { }); it('should build with cache and gain performance', async function() { + await t.destroy(apos); await removeCache(); await removeCache(cacheFolderPath.replace('/webpack-cache', '/changed')); @@ -346,9 +408,11 @@ describe('Assets', function() { assert(meta2['default:apos']); assert(meta2['default:src']); - // Expect at least 40% gain, in reallity it should be 50+ + // Caching should provide a measurable speedup. The threshold is kept + // low (10%) to avoid flaky failures on loaded CI runners where the + // cold run can be fast due to OS-level caching. const gain = (execTime - execTimeCached) / execTime * 100; - assert(gain >= 20, `Expected gain >=20%, got ${gain}%`); + assert(gain >= 10, `Expected gain >=10%, got ${gain}%`); // Modification times assert(meta['default:apos'].mdate); @@ -509,11 +573,11 @@ describe('Assets', function() { assert(apos.asset.restartId); assert(!result.builds); assert(!result.changes); + await waitForWatcherReady(apos.asset.buildWatcher); // Modify asset and rebuild const assetPath = path.join(process.cwd(), 'test/modules/bundle-page/ui/src/extra.js'); const assetPathPublic = path.join(process.cwd(), 'test/public/apos-frontend/default/extra-module-bundle.js'); - const assetContent = fs.readFileSync(assetPath, 'utf-8'); fs.writeFileSync( assetPath, 'export default () => { \'bundle-page-watcher-test-src\'; };\n', @@ -524,34 +588,33 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathPublic, 'utf8')).match(/bundle-page-watcher-test-src/), 'Unable to verify public asset was rebuilt by the watcher', 500, - 10000 + 20000 ); await retryAssertTrue( () => apos.asset.restartId !== restartId, 'Unable to verify restartId has been changed', 500, - 10000 + 20000 ); await retryAssertTrue( () => result.builds.length === 1 && result.builds.includes('src'), 'Unable to verify build "src" has been triggered', 50, - 1000 + 2000 ); await retryAssertTrue( () => result.changes.length === 1 && result.changes[0].includes('modules/bundle-page/ui/src/extra.js'), 'Unable to verify changes contain the proper file', 50, - 1000 + 2000 ); await t.destroy(apos); assert.equal(apos.asset.buildWatcher, null); apos = null; - fs.writeFileSync(assetPath, assetContent, 'utf8'); }); it('should watch and rebuild assets and reload page in development (src)', async function() { @@ -567,12 +630,6 @@ describe('Assets', function() { const assetPathPublicCss = path.join(rootPath, 'test/public/apos-frontend/default/public-bundle.css'); const assetPathAposJs = path.join(rootPath, 'test/public/apos-frontend/default/apos-module-bundle.js'); const assetPathAposCss = path.join(rootPath, 'test/public/apos-frontend/default/apos-bundle.css'); - const assetContentJs = fs.readFileSync(assetPathJs, 'utf-8'); - const assetContentScss = fs.readFileSync(assetPathScss, 'utf-8'); - // Resurrect the default assets content if test has failed - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); - fs.writeFileSync(assetPathScss, assetContentScss, 'utf8'); - apos = await t.create({ root: module, autoBuild: true, @@ -595,6 +652,7 @@ describe('Assets', function() { assert(apos.asset.restartId); assert(!result.builds); assert(!result.changes); + await waitForWatcherReady(apos.asset.buildWatcher); // * modify assets and rebuild fs.writeFileSync( @@ -613,13 +671,13 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathPublicJs, 'utf8')).match(/default-page-watcher-test-src/), 'Unable to verify public JS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); await retryAssertTrue( async () => (await fs.readFile(assetPathPublicCss, 'utf8')).match(/\.default-page-watcher-test-src/), 'Unable to verify public CSS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); // * change is in the apos bundle @@ -627,13 +685,13 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathAposJs, 'utf8')).match(/default-page-watcher-test-src/), 'Unable to verify apos JS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); await retryAssertTrue( async () => (await fs.readFile(assetPathAposCss, 'utf8')).match(/\.default-page-watcher-test-src/), 'Unable to verify apos CSS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); // * page has been restarted @@ -641,7 +699,7 @@ describe('Assets', function() { () => apos.asset.restartId !== restartId, 'Unable to verify restartId has been changed', 500, - 10000 + 20000 ); // * only src related builds were triggered @@ -650,7 +708,7 @@ describe('Assets', function() { result.builds.includes('src'), 'Unable to verify build "src" has been triggered', 50, - 1000 + 2000 ); // * changes detected @@ -665,14 +723,12 @@ describe('Assets', function() { .length === 2, 'Unable to verify changes contain the proper source files', 50, - 1000 + 2000 ); await t.destroy(apos); assert.equal(apos.asset.buildWatcher, null); apos = null; - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); - fs.writeFileSync(assetPathScss, assetContentScss, 'utf8'); }); it('should watch and rebuild assets and reload page in development (public)', async function() { @@ -688,12 +744,6 @@ describe('Assets', function() { const assetPathPublicCss = path.join(rootPath, 'test/public/apos-frontend/default/public-bundle.css'); const assetPathAposJs = path.join(rootPath, 'test/public/apos-frontend/default/apos-module-bundle.js'); const assetPathAposCss = path.join(rootPath, 'test/public/apos-frontend/default/apos-bundle.css'); - const assetContentJs = fs.readFileSync(assetPathJs, 'utf-8'); - const assetContentScss = fs.readFileSync(assetPathCss, 'utf-8'); - // Resurrect the default assets content if test has failed - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); - fs.writeFileSync(assetPathCss, assetContentScss, 'utf8'); - apos = await t.create({ root: module, autoBuild: true, @@ -716,6 +766,7 @@ describe('Assets', function() { assert(apos.asset.restartId); assert(!result.builds); assert(!result.changes); + await waitForWatcherReady(apos.asset.buildWatcher); // * modify assets and rebuild fs.writeFileSync( @@ -734,13 +785,13 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathPublicJs, 'utf8')).match(/default-page-watcher-test-public/), 'Unable to verify public JS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); await retryAssertTrue( async () => (await fs.readFile(assetPathPublicCss, 'utf8')).match(/\.default-page-watcher-test-public/), 'Unable to verify public CSS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); // * change is in the apos bundle @@ -748,13 +799,13 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathAposJs, 'utf8')).match(/default-page-watcher-test-public/), 'Unable to verify apos JS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); await retryAssertTrue( async () => (await fs.readFile(assetPathAposCss, 'utf8')).match(/\.default-page-watcher-test-public/), 'Unable to verify apos CSS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); // * page has been restarted @@ -762,7 +813,7 @@ describe('Assets', function() { () => apos.asset.restartId !== restartId, 'Unable to verify restartId has been changed', 500, - 10000 + 20000 ); // * only public build was triggered @@ -771,7 +822,7 @@ describe('Assets', function() { result.builds.includes('public'), 'Unable to verify build "public" has been triggered', 50, - 1000 + 2000 ); // * changes detected @@ -786,14 +837,12 @@ describe('Assets', function() { .length === 2, 'Unable to verify changes contain the proper source files', 50, - 1000 + 2000 ); await t.destroy(apos); assert.equal(apos.asset.buildWatcher, null); apos = null; - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); - fs.writeFileSync(assetPathCss, assetContentScss, 'utf8'); }); it('should watch and rebuild assets and reload page in development (apos)', async function() { @@ -836,6 +885,7 @@ describe('Assets', function() { assert(apos.asset.restartId); assert(!result.builds); assert(!result.changes); + await waitForWatcherReady(apos.asset.buildWatcher); // * modify assets and rebuild fs.writeFileSync( @@ -850,7 +900,7 @@ describe('Assets', function() { .includes('default-page-watcher-test-apos'), 'Unable to verify apos JS asset was rebuilt by the watcher', 500, - 20000 + 40000 ); // * page has been restarted @@ -858,7 +908,7 @@ describe('Assets', function() { () => apos.asset.restartId !== restartId, 'Unable to verify restartId has been changed', 500, - 10000 + 20000 ); // * only apos build was triggered @@ -867,7 +917,7 @@ describe('Assets', function() { result.builds.includes('apos'), 'Unable to verify build "apos" has been triggered', 50, - 1000 + 2000 ); // * changes detected @@ -877,13 +927,12 @@ describe('Assets', function() { result.changes[0].includes('modules/default-page/ui/apos/components/FakeComponent.vue'), 'Unable to verify changes contain the proper source files', 50, - 1000 + 2000 ); await t.destroy(apos); assert.equal(apos.asset.buildWatcher, null); apos = null; - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); }); it('should watch and recover after build error in development', async function() { @@ -898,9 +947,6 @@ describe('Assets', function() { const assetPathScss = path.join(rootPath, 'test/modules/default-page/ui/src/index.scss'); const assetPathPublicCss = path.join(rootPath, 'test/public/apos-frontend/default/public-bundle.css'); const assetPathAposCss = path.join(rootPath, 'test/public/apos-frontend/default/apos-bundle.css'); - const assetContentScss = '.default-page {color:red;}\n'; - // Resurrect the default assets content if test has failed - fs.writeFileSync(assetPathScss, assetContentScss, 'utf8'); apos = await t.create({ root: module, @@ -924,6 +970,7 @@ describe('Assets', function() { assert(apos.asset.restartId); assert(!result.builds); assert(!result.changes); + await waitForWatcherReady(apos.asset.buildWatcher); // * modify assets and rebuild fs.writeFileSync( @@ -937,7 +984,7 @@ describe('Assets', function() { () => called === 1 && result.builds.length === 0, 'Unable to verify build with error was triggered', 100, - 10000 + 20000 ); // * page has NOT been restarted @@ -945,7 +992,7 @@ describe('Assets', function() { () => apos.asset.restartId === restartId, 'Unable to verify restartId has been changed', 100, - 10000 + 20000 ); // * modify assets and recover @@ -960,7 +1007,7 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathPublicCss, 'utf8')).match(/\.default-page-watcher-test-recover/), 'Unable to verify public CSS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); // * change is in the apos bundle @@ -968,7 +1015,7 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathAposCss, 'utf8')).match(/\.default-page-watcher-test-recover/), 'Unable to verify apos CSS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); // * page has been restarted @@ -976,7 +1023,7 @@ describe('Assets', function() { () => apos.asset.restartId !== restartId, 'Unable to verify restartId has been changed', 500, - 10000 + 20000 ); // * only src related builds were triggered @@ -985,7 +1032,7 @@ describe('Assets', function() { result.builds.includes('src'), 'Unable to verify build "src" have been triggered', 50, - 1000 + 2000 ); // * changes detected @@ -999,13 +1046,12 @@ describe('Assets', function() { .length === 1, 'Unable to verify changes contain the proper source files', 50, - 1000 + 2000 ); await t.destroy(apos); assert.equal(apos.asset.buildWatcher, null); apos = null; - fs.writeFileSync(assetPathScss, assetContentScss, 'utf8'); }); it('should watch but not rebuild assets and not reload page when changes are not in use', async function() { @@ -1023,12 +1069,6 @@ describe('Assets', function() { const assetPathPublicCss = path.join(rootPath, 'test/public/apos-frontend/default/public-bundle.css'); const assetPathAposJs = path.join(rootPath, 'test/public/apos-frontend/default/apos-module-bundle.js'); const assetPathAposCss = path.join(rootPath, 'test/public/apos-frontend/default/apos-bundle.css'); - const assetContentJs = fs.readFileSync(assetPathJs, 'utf-8'); - const assetContentScss = fs.readFileSync(assetPathScss, 'utf-8'); - // Resurrect the default assets content if test has failed - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); - fs.writeFileSync(assetPathScss, assetContentScss, 'utf8'); - apos = await t.create({ root: module, autoBuild: true, @@ -1051,6 +1091,7 @@ describe('Assets', function() { assert(!result.builds); assert(!result.changes); assert.equal(rebuilt, false); + await waitForWatcherReady(apos.asset.buildWatcher); // * modify assets fs.writeFileSync( @@ -1139,8 +1180,6 @@ describe('Assets', function() { await t.destroy(apos); assert.equal(apos.asset.buildWatcher, null); apos = null; - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); - fs.writeFileSync(assetPathScss, assetContentScss, 'utf8'); }); it('should watch and rebuild assets in a debounced queue', async function() { @@ -1167,10 +1206,10 @@ describe('Assets', function() { } }); assert(apos.asset.buildWatcher); + await waitForWatcherReady(apos.asset.buildWatcher); const assetPath = path.join(process.cwd(), 'test/modules/bundle-page/ui/src/extra.js'); const assetPathPublic = path.join(process.cwd(), 'test/public/apos-frontend/default/extra-module-bundle.js'); - const assetContent = fs.readFileSync(assetPath, 'utf-8'); // Modify below the debounce rate for (const i of [ 1, 2, 3 ]) { @@ -1195,7 +1234,9 @@ describe('Assets', function() { 5000 ); - // Modify above the debounce rate, test the queue cap + // Modify well above the debounce rate (default 1000ms) so each + // write triggers its own rebuild. Use 2000ms to avoid flaky + // failures on loaded CI runners. timesRebuilt = 0; for (const i of [ 1, 2, 3 ]) { await fs.writeFile( @@ -1203,7 +1244,7 @@ describe('Assets', function() { `export default () => { 'bundle-page-watcher-test-${i}0'; };\n`, 'utf8' ); - await Promise.delay(1050); + await Promise.delay(2000); } await retryAssertTrue( async () => (await fs.readFile(assetPathPublic, 'utf8')).match(/bundle-page-watcher-test-30/), @@ -1220,10 +1261,10 @@ describe('Assets', function() { await t.destroy(apos); apos = null; - fs.writeFileSync(assetPath, assetContent, 'utf8'); }); it('should be able to setup the debounce time', async function() { + await t.destroy(apos); apos = await t.create({ root: module, @@ -1317,6 +1358,7 @@ describe('Assets', function() { }); it('should pass the right options to webpack extensions from all modules', async function() { + await t.destroy(apos); const { extConfig1, extConfig2 } = getWebpackConfigsForExtensionOptions(); apos = await t.create({ @@ -1347,6 +1389,7 @@ describe('Assets', function() { }); it('should allow two modules extending each others to pass options to the same webpack extension', async function() { + await t.destroy(apos); const { extConfig1, extConfig2 } = getWebpackConfigsForExtensionOptions(); apos = await t.create({ diff --git a/packages/apostrophe/test/db-tools.js b/packages/apostrophe/test/db-tools.js new file mode 100644 index 0000000000..2bc33de36a --- /dev/null +++ b/packages/apostrophe/test/db-tools.js @@ -0,0 +1,365 @@ +const assert = require('assert'); +const { execFile } = require('child_process'); +const path = require('path'); +const fs = require('fs'); +const os = require('os'); +const dbConnect = require('@apostrophecms/db-connect'); + +const dumpBin = require.resolve('@apostrophecms/db-connect/bin/apos-db-dump.js'); +const restoreBin = require.resolve('@apostrophecms/db-connect/bin/apos-db-restore.js'); + +const testDbProtocol = process.env.APOS_TEST_DB_PROTOCOL || 'mongodb'; + +function testUri(dbName) { + const dbSafe = dbName.replace(/-/g, '_').replace(/[^a-zA-Z0-9_]/g, ''); + if (testDbProtocol === 'sqlite') { + return `sqlite://${path.join(os.tmpdir(), `${dbSafe}.db`)}`; + } + if (testDbProtocol === 'postgres') { + return `postgres://localhost:5432/${dbSafe}`; + } + const baseUri = process.env.DB_URI || 'mongodb://localhost:27017'; + return `${baseUri}/${dbName}`; +} + +function run(bin, args) { + return new Promise((resolve, reject) => { + execFile(process.execPath, [ bin, ...args ], { + timeout: 30000, + maxBuffer: 50 * 1024 * 1024 + }, (err, stdout, stderr) => { + if (err) { + err.stdout = stdout; + err.stderr = stderr; + return reject(err); + } + resolve({ + stdout, + stderr + }); + }); + }); +} + +async function dropAll(uri) { + let client; + try { + client = await dbConnect(uri); + } catch (e) { + return; + } + const db = client.db(); + const collections = await db.listCollections().toArray(); + for (const col of collections) { + await db.collection(col.name).drop(); + } + await client.close(); +} + +describe('apos-db-dump and apos-db-restore', function () { + this.timeout(30000); + + const sourceUri = testUri('dbtest_dump_source'); + const targetUri = testUri('dbtest_dump_target'); + let tmpFile; + + before(async function () { + tmpFile = path.join(os.tmpdir(), `apos-db-test-${process.pid}.ndjson`); + await dropAll(sourceUri); + await dropAll(targetUri); + }); + + after(async function () { + await dropAll(sourceUri); + await dropAll(targetUri); + try { + fs.unlinkSync(tmpFile); + } catch (e) { + // ignore + } + }); + + it('should dump an empty database without error', async function () { + const { stdout } = await run(dumpBin, [ sourceUri ]); + assert.strictEqual(stdout.trim(), ''); + }); + + it('should dump and restore documents', async function () { + // Insert test data + const client = await dbConnect(sourceUri); + const db = client.db(); + await db.collection('aposDocs').insertMany([ + { + _id: 'doc1', + title: 'Hello', + tags: [ 'a', 'b' ] + }, + { + _id: 'doc2', + title: 'World' + } + ]); + await db.collection('aposCache').insertMany([ + { + _id: 'cache1', + value: 42 + } + ]); + await client.close(); + + // Dump to file + await run(dumpBin, [ sourceUri, `--output=${tmpFile}` ]); + const content = fs.readFileSync(tmpFile, 'utf8'); + const lines = content.split('\n').filter(l => l.trim()); + + // Should have header + docs for each collection + assert(lines.length >= 4, `Expected at least 4 lines, got ${lines.length}`); + + // Every line should be valid JSON + for (const line of lines) { + JSON.parse(line); + } + + // Should have collection headers + const headers = lines + .map(l => JSON.parse(l)) + .filter(e => e._collection && !e._doc); + const collNames = headers.map(h => h._collection).sort(); + assert(collNames.includes('aposDocs')); + assert(collNames.includes('aposCache')); + + // Restore to target + await run(restoreBin, [ targetUri, `--input=${tmpFile}` ]); + + // Verify target has the data + const client2 = await dbConnect(targetUri); + const db2 = client2.db(); + const docs = await db2.collection('aposDocs').find({}).sort({ _id: 1 }).toArray(); + assert.strictEqual(docs.length, 2); + assert.strictEqual(docs[0]._id, 'doc1'); + assert.strictEqual(docs[0].title, 'Hello'); + assert.deepStrictEqual(docs[0].tags, [ 'a', 'b' ]); + assert.strictEqual(docs[1]._id, 'doc2'); + + const cacheDoc = await db2.collection('aposCache').findOne({ _id: 'cache1' }); + assert(cacheDoc); + assert.strictEqual(cacheDoc.value, 42); + await client2.close(); + }); + + it('should preserve Date objects via $date serialization', async function () { + await dropAll(sourceUri); + const client = await dbConnect(sourceUri); + const db = client.db(); + const testDate = new Date('2024-06-15T10:30:00.000Z'); + await db.collection('aposDocs').insertOne({ + _id: 'dateDoc', + createdAt: testDate, + nested: { updatedAt: testDate } + }); + await client.close(); + + // Dump and check format + const { stdout } = await run(dumpBin, [ sourceUri ]); + assert(stdout.includes('"$date"')); + assert(stdout.includes('2024-06-15T10:30:00.000Z'), 'Should contain ISO date string'); + + // Restore and verify dates come back as Date objects + await dropAll(targetUri); + await run(dumpBin, [ sourceUri, `--output=${tmpFile}` ]); + await run(restoreBin, [ targetUri, `--input=${tmpFile}` ]); + + const client2 = await dbConnect(targetUri); + const db2 = client2.db(); + const doc = await db2.collection('aposDocs').findOne({ _id: 'dateDoc' }); + assert(doc.createdAt instanceof Date); + assert.strictEqual(doc.createdAt.toISOString(), '2024-06-15T10:30:00.000Z'); + assert(doc.nested.updatedAt instanceof Date); + assert.strictEqual(doc.nested.updatedAt.toISOString(), '2024-06-15T10:30:00.000Z'); + await client2.close(); + }); + + it('should dump and restore indexes', async function () { + await dropAll(sourceUri); + const client = await dbConnect(sourceUri); + const db = client.db(); + const col = db.collection('aposDocs'); + await col.insertMany([ + { + _id: 'idx1', + slug: 'hello', + price: 10 + }, + { + _id: 'idx2', + slug: 'world', + price: 20 + } + ]); + await col.createIndex({ slug: 1 }); + await col.createIndex({ slug: 1 }, { + unique: true, + name: 'slug_unique' + }); + await col.createIndex({ price: 1 }, { type: 'number' }); + await client.close(); + + // Dump + await run(dumpBin, [ sourceUri, `--output=${tmpFile}` ]); + + const content = fs.readFileSync(tmpFile, 'utf8'); + const header = JSON.parse(content.split('\n')[0]); + assert(header._indexes, 'Header should contain _indexes'); + assert(header._indexes.length >= 2, 'Should have at least 2 custom indexes'); + + // Restore + await dropAll(targetUri); + await run(restoreBin, [ targetUri, `--input=${tmpFile}` ]); + + // Verify indexes exist on target + const client2 = await dbConnect(targetUri); + const db2 = client2.db(); + const indexes = await db2.collection('aposDocs').indexes(); + assert(indexes.find(i => i.key && i.key.slug === 1 && !i.unique), + 'Should have regular slug index'); + assert(indexes.find(i => i.key && i.key.slug === 1 && i.unique), + 'Should have unique slug index'); + + // Verify unique constraint is enforced + try { + await db2.collection('aposDocs').insertOne({ + _id: 'idx3', + slug: 'hello' + }); + assert.fail('Should have rejected duplicate slug'); + } catch (e) { + assert(e.code === 11000 || /duplicate|unique|already exists/i.test(e.message)); + } + + await client2.close(); + }); + + it('should handle piped stdout-to-stdin', async function () { + await dropAll(sourceUri); + const client = await dbConnect(sourceUri); + const db = client.db(); + await db.collection('aposDocs').insertMany([ + { + _id: 'pipe1', + title: 'Piped' + } + ]); + await client.close(); + + // Dump to file, then restore from file (simulating pipe) + const { stdout } = await run(dumpBin, [ sourceUri ]); + + // Write stdout to tmp, restore from it + fs.writeFileSync(tmpFile, stdout); + await dropAll(targetUri); + await run(restoreBin, [ targetUri, `--input=${tmpFile}` ]); + + const client2 = await dbConnect(targetUri); + const db2 = client2.db(); + const doc = await db2.collection('aposDocs').findOne({ _id: 'pipe1' }); + assert(doc); + assert.strictEqual(doc.title, 'Piped'); + await client2.close(); + }); + + it('should handle large collections in batches', async function () { + await dropAll(sourceUri); + const client = await dbConnect(sourceUri); + const db = client.db(); + const docs = []; + for (let i = 0; i < 350; i++) { + docs.push({ + _id: `batch${String(i).padStart(4, '0')}`, + value: i + }); + } + await db.collection('aposDocs').insertMany(docs); + await client.close(); + + // Dump + await run(dumpBin, [ sourceUri, `--output=${tmpFile}` ]); + + const content = fs.readFileSync(tmpFile, 'utf8'); + const lines = content.split('\n').filter(l => l.trim()); + // 1 header + 350 doc lines + assert.strictEqual(lines.length, 351); + + // Docs should be sorted by _id + const docLines = lines.slice(1).map(l => JSON.parse(l)); + for (let i = 1; i < docLines.length; i++) { + assert(docLines[i]._doc._id > docLines[i - 1]._doc._id, + 'Docs should be sorted by _id'); + } + + // Restore and verify count + await dropAll(targetUri); + await run(restoreBin, [ targetUri, `--input=${tmpFile}` ]); + + const client2 = await dbConnect(targetUri); + const db2 = client2.db(); + const count = await db2.collection('aposDocs').countDocuments({}); + assert.strictEqual(count, 350); + await client2.close(); + }); + + it('should restore to a clean state (drop existing data)', async function () { + // Put some pre-existing data in target + const client = await dbConnect(targetUri); + const db = client.db(); + try { + await db.collection('aposDocs').drop(); + } catch (e) { + // ignore + } + await db.collection('aposDocs').insertOne({ + _id: 'old', + title: 'Should be removed' + }); + await client.close(); + + // Set up source with different data + await dropAll(sourceUri); + const client2 = await dbConnect(sourceUri); + const db2 = client2.db(); + await db2.collection('aposDocs').insertOne({ + _id: 'new', + title: 'Fresh data' + }); + await client2.close(); + + // Dump source and restore to target + await run(dumpBin, [ sourceUri, `--output=${tmpFile}` ]); + await run(restoreBin, [ targetUri, `--input=${tmpFile}` ]); + + // Target should only have the new data + const client3 = await dbConnect(targetUri); + const db3 = client3.db(); + const all = await db3.collection('aposDocs').find({}).toArray(); + assert.strictEqual(all.length, 1); + assert.strictEqual(all[0]._id, 'new'); + await client3.close(); + }); + + it('should fail with usage error when no URI is provided', async function () { + try { + await run(dumpBin, []); + assert.fail('Should have exited with error'); + } catch (e) { + assert.strictEqual(e.code, 1); + assert(e.stderr.includes('Usage')); + } + + try { + await run(restoreBin, []); + assert.fail('Should have exited with error'); + } catch (e) { + assert.strictEqual(e.code, 1); + assert(e.stderr.includes('Usage')); + } + }); +}); diff --git a/packages/apostrophe/test/db.js b/packages/apostrophe/test/db.js index 47afa4cf16..7a6134b4de 100644 --- a/packages/apostrophe/test/db.js +++ b/packages/apostrophe/test/db.js @@ -1,6 +1,10 @@ const t = require('../test-lib/test.js'); const assert = require('assert'); +const bogusUri = t.testDbProtocol === 'postgres' + ? 'postgres://this-will-not-work-unless-db-successfully-overrides-it/fail' + : 'mongodb://this-will-not-work-unless-db-successfully-overrides-it/fail'; + describe('Db', function() { let apos, apos2; @@ -25,23 +29,28 @@ describe('Db', function() { assert(doc); }); - it('should be able to launch a second instance reusing the connection', async function() { - // Often takes too long otherwise - this.timeout(10000); - apos2 = await t.create({ - root: module, - modules: { - '@apostrophecms/db': { - options: { - client: apos.dbClient, - uri: 'mongodb://this-will-not-work-unless-db-successfully-overrides-it/fail' + // Client reuse with a different database name is only supported in + // mongodb and multipostgres mode, not simple postgres (which has no + // schema isolation) + if (t.testDbProtocol !== 'postgres') { + it('should be able to launch a second instance reusing the connection', async function() { + // Often takes too long otherwise + this.timeout(10000); + apos2 = await t.create({ + root: module, + modules: { + '@apostrophecms/db': { + options: { + client: apos.dbClient, + uri: bogusUri + } } } - } - }); + }); - const doc = await apos2.doc.db.findOne(); + const doc = await apos2.doc.db.findOne(); - assert(doc); - }); + assert(doc); + }); + } }); diff --git a/packages/apostrophe/test/default-adapter.js b/packages/apostrophe/test/default-adapter.js new file mode 100644 index 0000000000..ad8bc628e8 --- /dev/null +++ b/packages/apostrophe/test/default-adapter.js @@ -0,0 +1,256 @@ +const assert = require('assert'); +const path = require('path'); + +describe('Default Adapter', function() { + + this.timeout(20000); + + // Save and restore env vars + let savedAposDefaultDbAdapter; + let savedAposDbUri; + let savedAposMongdbUri; + + beforeEach(function() { + savedAposDefaultDbAdapter = process.env.APOS_DEFAULT_DB_ADAPTER; + savedAposDbUri = process.env.APOS_DB_URI; + savedAposMongdbUri = process.env.APOS_MONGODB_URI; + delete process.env.APOS_DEFAULT_DB_ADAPTER; + delete process.env.APOS_DB_URI; + delete process.env.APOS_MONGODB_URI; + }); + + afterEach(function() { + if (savedAposDefaultDbAdapter !== undefined) { + process.env.APOS_DEFAULT_DB_ADAPTER = savedAposDefaultDbAdapter; + } else { + delete process.env.APOS_DEFAULT_DB_ADAPTER; + } + if (savedAposDbUri !== undefined) { + process.env.APOS_DB_URI = savedAposDbUri; + } else { + delete process.env.APOS_DB_URI; + } + if (savedAposMongdbUri !== undefined) { + process.env.APOS_MONGODB_URI = savedAposMongdbUri; + } else { + delete process.env.APOS_MONGODB_URI; + } + }); + + // These tests verify URI construction by examining module internals + // without needing actual database connections. + + it('builds mongodb:// URI by default', function() { + const uri = buildUri({ shortName: 'mysite' }); + assert(uri.startsWith('mongodb://')); + assert(uri.includes('localhost:27017')); + assert(uri.includes('/mysite')); + }); + + it('builds mongodb:// URI when defaultAdapter is "mongodb"', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { defaultAdapter: 'mongodb' } + }); + assert(uri.startsWith('mongodb://')); + assert(uri.includes('/mysite')); + }); + + it('builds sqlite:// URI when defaultAdapter is "sqlite"', function() { + const uri = buildUri({ + shortName: 'mysite', + rootDir: '/app', + dbOptions: { defaultAdapter: 'sqlite' } + }); + assert(uri.startsWith('sqlite://')); + assert(uri.includes(path.join('data', 'mysite.sqlite'))); + }); + + it('builds postgres:// URI when defaultAdapter is "postgres"', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { defaultAdapter: 'postgres' } + }); + assert(uri.startsWith('postgres://')); + assert(uri.includes('localhost:5432')); + assert(uri.includes('/mysite')); + }); + + it('builds multipostgres:// URI when defaultAdapter is "multipostgres"', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { defaultAdapter: 'multipostgres' } + }); + assert(uri.startsWith('multipostgres://')); + assert(uri.includes('localhost:5432')); + assert(uri.includes('/mysite')); + }); + + it('includes URI-encoded credentials for postgres', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { + defaultAdapter: 'postgres', + user: 'admin', + password: 'p@ss:word/special' + } + }); + assert(uri.startsWith('postgres://')); + assert(uri.includes('admin')); + assert(uri.includes(encodeURIComponent('p@ss:word/special'))); + assert(!uri.includes('p@ss:word/special')); + }); + + it('includes URI-encoded credentials for mongodb', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { + defaultAdapter: 'mongodb', + user: 'admin', + password: 'p@ss:word' + } + }); + assert(uri.startsWith('mongodb://')); + assert(uri.includes(encodeURIComponent('p@ss:word'))); + }); + + it('APOS_DEFAULT_DB_ADAPTER env var overrides the option', function() { + process.env.APOS_DEFAULT_DB_ADAPTER = 'postgres'; + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { defaultAdapter: 'mongodb' } + }); + assert(uri.startsWith('postgres://')); + }); + + it('throws for invalid adapter name', function() { + assert.throws(() => { + buildUri({ + shortName: 'mysite', + dbOptions: { defaultAdapter: 'invalid' } + }); + }, /Invalid defaultAdapter/); + }); + + it('explicit uri option overrides defaultAdapter', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { + defaultAdapter: 'postgres', + uri: 'mongodb://custom:27017/other' + } + }); + assert.strictEqual(uri, 'mongodb://custom:27017/other'); + }); + + it('APOS_DB_URI env var overrides everything', function() { + process.env.APOS_DB_URI = 'mongodb://envhost:27017/envdb'; + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { defaultAdapter: 'postgres' } + }); + assert.strictEqual(uri, 'mongodb://envhost:27017/envdb'); + }); + + it('honors custom host and port for postgres', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { + defaultAdapter: 'postgres', + host: 'dbserver', + port: 5433 + } + }); + assert(uri.includes('dbserver:5433')); + }); + + it('honors custom host and port for mongodb', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { + defaultAdapter: 'mongodb', + host: 'mongohost', + port: 27018 + } + }); + assert(uri.includes('mongohost:27018')); + }); + + it('uses shortName as database name by default', function() { + const uri = buildUri({ shortName: 'my-app' }); + assert(uri.endsWith('/my-app')); + }); + + it('uses name option over shortName when provided', function() { + const uri = buildUri({ + shortName: 'my-app', + dbOptions: { name: 'custom-db' } + }); + assert(uri.includes('/custom-db')); + }); + + it('ignores host/port/user/password for sqlite', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { + defaultAdapter: 'sqlite', + host: 'shouldbeignored', + port: 9999, + user: 'nobody', + password: 'nothing' + } + }); + assert(uri.startsWith('sqlite://')); + assert(!uri.includes('shouldbeignored')); + assert(!uri.includes('9999')); + assert(!uri.includes('nobody')); + }); +}); + +// Helper: simulate the URI construction logic from the db module +// without actually connecting. This extracts the same logic path. +function buildUri(options = {}) { + const escapeHost = require('../lib/escape-host.js'); + const shortName = options.shortName || 'test-app'; + const rootDir = options.rootDir || '/tmp/test-app'; + const dbOptions = { ...(options.dbOptions || {}) }; + + // Simulate the connectToDb URI construction + const viaEnv = process.env.APOS_DB_URI || process.env.APOS_MONGODB_URI; + if (viaEnv) { + return viaEnv; + } + if (dbOptions.uri) { + return dbOptions.uri; + } + + const validAdapters = [ 'mongodb', 'sqlite', 'postgres', 'multipostgres' ]; + const adapter = process.env.APOS_DEFAULT_DB_ADAPTER || dbOptions.defaultAdapter || 'mongodb'; + if (!validAdapters.includes(adapter)) { + throw new Error(`Invalid defaultAdapter: "${adapter}". Must be one of: ${validAdapters.join(', ')}`); + } + + if (!dbOptions.name) { + dbOptions.name = shortName; + } + + if (adapter === 'sqlite') { + const path = require('path'); + return `sqlite://${path.resolve(rootDir, 'data', dbOptions.name + '.sqlite')}`; + } + + const credentials = dbOptions.user + ? encodeURIComponent(dbOptions.user) + ':' + encodeURIComponent(dbOptions.password) + '@' + : ''; + + if (adapter === 'mongodb') { + const host = dbOptions.host || 'localhost'; + const port = dbOptions.port || 27017; + return 'mongodb://' + credentials + escapeHost(host) + ':' + port + '/' + dbOptions.name; + } + + // postgres or multipostgres + const host = dbOptions.host || 'localhost'; + const port = dbOptions.port || 5432; + return adapter + '://' + credentials + escapeHost(host) + ':' + port + '/' + dbOptions.name; +} diff --git a/packages/apostrophe/test/job.js b/packages/apostrophe/test/job.js index 75b1ad47b9..803368256e 100644 --- a/packages/apostrophe/test/job.js +++ b/packages/apostrophe/test/job.js @@ -311,7 +311,7 @@ describe('Job module', function() { req, async function(_req, reporters) { let count = 1; - reporters.setTotal(articleIds.length); + await reporters.setTotal(articleIds.length); for (const id of articleIds) { await delay(3); diff --git a/packages/broadband/package.json b/packages/broadband/package.json index 63400ffb0c..83ee406f23 100644 --- a/packages/broadband/package.json +++ b/packages/broadband/package.json @@ -32,6 +32,9 @@ "dependencies": { "lodash": "^4.18.1" }, + "apostropheTestConfig": { + "mongodbOnly": true + }, "devDependencies": { "eslint": "^9.39.1", "eslint-config-apostrophe": "workspace:^", diff --git a/packages/cache-on-demand/package.json b/packages/cache-on-demand/package.json index 9df4c1297a..7fec6b29ff 100644 --- a/packages/cache-on-demand/package.json +++ b/packages/cache-on-demand/package.json @@ -30,6 +30,9 @@ "bugs": { "url": "https://github.com/punkave/cache-on-demand/issues" }, + "apostropheTestConfig": { + "requiresMongo": false + }, "devDependencies": { "eslint": "^9.39.1", "eslint-config-apostrophe": "workspace:^", diff --git a/packages/db-connect/LICENSE.md b/packages/db-connect/LICENSE.md new file mode 100644 index 0000000000..7a8ddbb5ed --- /dev/null +++ b/packages/db-connect/LICENSE.md @@ -0,0 +1,7 @@ +Copyright (c) 2025 Apostrophe Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/packages/db-connect/README.md b/packages/db-connect/README.md new file mode 100644 index 0000000000..b1cf3aa6e6 --- /dev/null +++ b/packages/db-connect/README.md @@ -0,0 +1,107 @@ +# @apostrophecms/db-connect + +`@apostrophecms/db-connect` defines the database connection API for [ApostropheCMS](https://apostrophecms.com). It provides adapters for MongoDB, PostgreSQL, and SQLite, and includes the `apos-db-dump` and `apos-db-restore` command-line utilities for database migration and backup. + +The db-connect API is compatible with a large subset of the MongoDB API. However, after this introductory note, this document describes the **db-connect API** on its own terms. For projects that need to work across all three databases, it is necessary to use only the functionality defined here. + +## Supported Connection URLs + +### MongoDB + +``` +mongodb://localhost:27017/mydb +mongodb+srv://user:pass@cluster.example.com/mydb +``` + +Standard MongoDB connection strings. The MongoDB adapter is a thin wrapper around the existing driver. + +### PostgreSQL + +``` +postgres://localhost:5432/mydb +``` + +Single-database mode. All collections are stored as tables in the `public` schema. + +### SQLite + +``` +sqlite:///path/to/database.db +``` + +File-based SQLite databases using `better-sqlite3`. In-memory databases +(`sqlite://:memory:`) are not supported — the adapter assumes a persistent +store suitable for hosting an ApostropheCMS site. + +### Multi-Schema PostgreSQL (multipostgres) + +``` +multipostgres://localhost:5432/shareddb-tenant1 +``` + +Designed for use with the ApostropheCMS [multisite](https://apostrophecms.com/extensions/multisite-2) module. In this mode, each site gets its own PostgreSQL schema within a single physical database. + +The URL path is split at the **last hyphen**: + +- Everything before the last hyphen is the real PostgreSQL database name (`shareddb`) +- Everything after is the schema name (`tenant1`) + +Each schema is created automatically on first use and dropped cleanly when the database is dropped. This provides true multi-tenant isolation — no cross-tenant data leakage — while sharing a single PostgreSQL instance efficiently. + +## Connecting + +```js +const connect = require('@apostrophecms/db-connect'); + +const client = await connect('postgres://localhost:5432/mydb'); +const db = client.db(); +const articles = db.collection('articles'); + +// Insert a document +const result = await articles.insertOne({ title: 'Hello', status: 'draft' }); + +// Query documents +const docs = await articles.find({ status: 'draft' }) + .sort({ title: 1 }) + .limit(10) + .toArray(); + +// Update a document +await articles.updateOne( + { _id: result.insertedId }, + { $set: { status: 'published' } } +); + +await client.close(); +``` + +`connect(uri)` returns a client. Call `client.db()` to get a database, then `db.collection(name)` to get a collection. + +## Atomicity + +`insertOne` and `deleteOne` are always atomic. + +`updateOne` without `upsert` is atomic when it uses only the following operators: + +- **`$set`**, **`$inc`**, **`$unset`**, **`$currentDate`** — always atomic +- **`$push`**, **`$pull`**, **`$addToSet`** — atomic when all values are **scalars** (strings, numbers, booleans). Object or array values fall back to read-modify-write. + +These operators can be freely combined in a single `updateOne` call and the entire update will be atomic. This covers the most common patterns: counters, status flags, timestamps, field cleanup, and adding/removing items to/from arrays. + +**The following operations are NOT guaranteed to be atomic** and use a read-modify-write pattern (the document is read, the update is applied in JavaScript, and the result is written back): + +- `updateOne` with **`$rename`** or **`upsert: true`** +- `updateOne` with `$push`, `$pull`, or `$addToSet` using **non-scalar** values +- **`updateMany`**, **`findOneAndUpdate`**, and **`replaceOne`** + +For operations that must be atomic and aren't covered above, use advisory locking (`apos.lock`) to serialize access. Apostrophe core already uses advisory locking where atomicity matters (e.g., `apos.lock.lock` around critical sections). + +## API Reference + +- [Database and Client](./docs/database.md) — `client.db(name)`, listing collections, dropping databases +- [Collection Methods](./docs/collections.md) — CRUD operations, cursors, and bulk writes +- [Query Operators](./docs/queries.md) — filtering documents with comparison, logical, element, and array operators +- [Update Operators](./docs/updates.md) — modifying documents with `$set`, `$inc`, `$push`, and more +- [Indexes](./docs/indexes.md) — creating and managing indexes, including numeric and date types +- [Aggregation](./docs/aggregation.md) — pipeline stages and group accumulators +- [Dump and Restore](./docs/dump-restore.md) — CLI tools and programmatic API for backup and migration diff --git a/packages/db-connect/adapters/mongodb.js b/packages/db-connect/adapters/mongodb.js new file mode 100644 index 0000000000..1bf260d273 --- /dev/null +++ b/packages/db-connect/adapters/mongodb.js @@ -0,0 +1,10 @@ +// MongoDB adapter - thin wrapper around existing driver compatibility stack +const mongodbConnect = require('../lib/mongodb-connect'); + +module.exports = { + name: 'mongodb', + protocols: [ 'mongodb', 'mongodb+srv' ], + async connect(uri, options) { + return mongodbConnect(uri, options); + } +}; diff --git a/packages/db-connect/adapters/postgres.js b/packages/db-connect/adapters/postgres.js new file mode 100644 index 0000000000..7e99980a21 --- /dev/null +++ b/packages/db-connect/adapters/postgres.js @@ -0,0 +1,2505 @@ +// PostgreSQL Adapter for MongoDB-compatible interface +// Stores documents as JSONB with _id as primary key + +const { Pool } = require('pg'); +const crypto = require('crypto'); +const { + serializeValue, + serializeDocument, + deserializeDocument, + getNestedField, + setNestedField, + deepEqual, + applyProjection, + applyUpdate, + extractAnchoredLiteralPrefix, + prefixUpperBound, + validateInteger +} = require('../lib/shared'); +const { AggregationCursor } = require('../lib/aggregation-cursor'); + +// ============================================================================= +// PROFILING: Accumulated timing data for performance analysis +// Enable with POSTGRES_PROFILE=1 environment variable +// Print report with: require('.../postgres').profileReport() +// ============================================================================= + +const PROFILING = !!process.env.POSTGRES_PROFILE; + +const profile = { + buildWhereClause: { + calls: 0, + totalMs: 0 + }, + buildOrderBy: { + calls: 0, + totalMs: 0 + }, + serializeDocument: { + calls: 0, + totalMs: 0 + }, + convertDates: { + calls: 0, + totalMs: 0 + }, + applyProjection: { + calls: 0, + totalMs: 0 + }, + applyUpdate: { + calls: 0, + totalMs: 0 + }, + pgQuery: { + calls: 0, + totalMs: 0 + }, + ensureTable: { + calls: 0, + totalMs: 0 + }, + findOne: { + calls: 0, + totalMs: 0 + }, + findToArray: { + calls: 0, + totalMs: 0 + }, + cursorNext: { + calls: 0, + totalMs: 0 + }, + updateOne: { + calls: 0, + totalMs: 0 + }, + insertOne: { + calls: 0, + totalMs: 0 + }, + countDocuments: { + calls: 0, + totalMs: 0 + }, + distinct: { + calls: 0, + totalMs: 0 + } +}; + +// Per-query tracking: SQL text -> { calls, totalMs } +const queryProfile = {}; + +function profileStart() { + if (!PROFILING) { + return 0; + } + return performance.now(); +} + +function profileEnd(category, start) { + if (!PROFILING) { + return; + } + const elapsed = performance.now() - start; + profile[category].calls++; + profile[category].totalMs += elapsed; +} + +function profileQuery(sql, start) { + if (!PROFILING) { + return; + } + const elapsed = performance.now() - start; + profile.pgQuery.calls++; + profile.pgQuery.totalMs += elapsed; + // Normalize SQL for grouping: collapse $N params and specific values + const normalized = sql.replace(/\$\d+/g, '$?').replace(/\s+/g, ' ').trim().substring(0, 120); + if (!queryProfile[normalized]) { + queryProfile[normalized] = { + calls: 0, + totalMs: 0 + }; + } + queryProfile[normalized].calls++; + queryProfile[normalized].totalMs += elapsed; +} + +function profileReport() { + console.log('\n=== PostgreSQL Adapter Profile ===\n'); + + // High-level categories + console.log('--- Cumulative time by category ---'); + const sorted = Object.entries(profile) + .filter(([ , v ]) => v.calls > 0) + .sort((a, b) => b[1].totalMs - a[1].totalMs); + for (const [ name, data ] of sorted) { + console.log(` ${name.padEnd(20)} ${data.totalMs.toFixed(1).padStart(8)}ms (${data.calls} calls, ${(data.totalMs / data.calls).toFixed(3)}ms avg)`); + } + + // Per-query breakdown + console.log('\n--- Top queries by total time ---'); + const querySorted = Object.entries(queryProfile) + .sort((a, b) => b[1].totalMs - a[1].totalMs) + .slice(0, 20); + for (const [ sql, data ] of querySorted) { + console.log(` ${data.totalMs.toFixed(1).padStart(8)}ms (${String(data.calls).padStart(4)} calls, ${(data.totalMs / data.calls).toFixed(3)}ms avg) ${sql}`); + } + + console.log('\n=== End Profile ===\n'); +} + +function profileReset() { + for (const key of Object.keys(profile)) { + profile[key].calls = 0; + profile[key].totalMs = 0; + } + for (const key of Object.keys(queryProfile)) { + delete queryProfile[key]; + } +} + +// ============================================================================= +// SECURITY: Input Validation and Escaping +// ============================================================================= + +// Strict pattern for table/index names (PostgreSQL identifier limitations) +const SAFE_IDENTIFIER_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + +// Validate and sanitize a table/collection name +function validateTableName(name) { + if (typeof name !== 'string' || name.length === 0 || name.length > 63) { + throw new Error('Invalid table name: must be a non-empty string up to 63 characters'); + } + // Replace hyphens with underscores (common in MongoDB collection names) + const sanitized = name.replace(/-/g, '_'); + if (!SAFE_IDENTIFIER_PATTERN.test(sanitized)) { + throw new Error(`Invalid table name: "${name}" contains disallowed characters`); + } + return sanitized; +} + +// Sanitize a caller-supplied index name so it is safe to use as a +// PostgreSQL identifier. Unlike validateTableName() (which rejects unsafe +// input as a security measure against malicious table names), index names +// frequently arrive from cross-backend JSONL dumps — MongoDB's +// default/auto-generated index names contain characters like "." that are +// illegal PostgreSQL identifiers. Silently replacing those characters +// with "_" is safe: the name is an internal identifier, not user data, +// and the adapter is the one consulting the _indexes map by the sanitized +// form. +function sanitizeIndexName(name) { + if (typeof name !== 'string' || name.length === 0) { + return null; + } + const truncated = name.substring(0, 63); + const sanitized = truncated.replace(/[^a-zA-Z0-9_]/g, '_'); + if (/^[0-9]/.test(sanitized)) { + return '_' + sanitized.substring(0, 62); + } + return sanitized; +} + +// Escape a PostgreSQL identifier (table name, index name) for use in double quotes +// PostgreSQL: double any internal double quotes +function escapeIdentifier(name) { + return name.replace(/"/g, '""'); +} + +// Escape a string for use in single quotes (JSON path segments) +// PostgreSQL: double any internal single quotes +function escapeString(str) { + return str.replace(/'/g, '\'\''); +} + +// Create a MongoDB-compatible duplicate key error from a PostgreSQL 23505 error +function makeDuplicateKeyError(pgError) { + let field = null; + let value = null; + // Parse PostgreSQL detail to extract keyValue for MongoDB compatibility + // Detail format: Key ((data ->> 'field'::text))=(value) already exists. + // or for _id: Key (_id)=(value) already exists. + // Note: PostgreSQL wraps expression indexes in double parens + if (pgError.detail) { + const match = pgError.detail.match(/Key \((.+?)\)=\((.+?)\) already exists/); + if (match) { + const expr = match[1]; + value = match[2]; + if (expr === '_id') { + field = '_id'; + } else { + // Try to map the PostgreSQL expression back to a field name + // e.g., (data ->> 'username'::text) -> username + const fieldMatch = expr.match(/>>\s*'([^']+)'/); + if (fieldMatch) { + field = fieldMatch[1]; + } + } + } + } + const message = field && value + ? `Duplicate key error: ${field} "${value}" already exists` + : 'Duplicate key error'; + const error = new Error(message); + error.code = 11000; + if (field && value) { + error.keyValue = { [field]: value }; + } + return error; +} + +// Generate a MongoDB-style ObjectId-like string +function generateId() { + return crypto.randomBytes(12).toString('hex'); +} + +// Parse a PostgreSQL index definition (from pg_indexes.indexdef) back into +// our abstract index metadata: { key, unique, sparse, type }. +// This only handles the patterns our own createIndex generates — it is not +// a general SQL parser. +function parseIndexDef(indexdef) { + const unique = /\bUNIQUE\b/.test(indexdef); + const sparse = /\bWHERE\b/.test(indexdef); + const isGin = /\bUSING gin\b/.test(indexdef); + + // Strip COLLATE "C" annotations so the existing expression patterns match + // regardless of whether the JSON path has an explicit collation (introduced + // to make byte-wise comparisons match MongoDB semantics). + indexdef = indexdef.replace(/\s*COLLATE\s+"[^"]+"/gi, ''); + + if (isGin) { + // Text index: USING gin(to_tsvector('simple', coalesce(data->>'field', '') ...)) + // PostgreSQL normalizes to: COALESCE((data ->> 'field'::text), ''::text) + const key = {}; + const fieldPattern = /coalesce\(\s*\(*(data(?:\s*->\s*'[^']*'(?:::text)?)*\s*->>\s*'[^']*'(?:::text)?)\)*\s*,\s*''(?:::text)?\s*\)/gi; + let m; + while ((m = fieldPattern.exec(indexdef)) !== null) { + const fieldName = jsonPathToFieldName(m[1]); + if (fieldName) { + key[fieldName] = 'text'; + } + } + return { + key, + unique, + ...(sparse ? { sparse: true } : {}) + }; + } + + // Regular or unique index: extract expressions from the column list + // The column list is inside the last pair of parentheses before an + // optional WHERE clause + let colSection = indexdef; + const wherePos = colSection.indexOf(' WHERE '); + if (wherePos !== -1) { + colSection = colSection.substring(0, wherePos); + } + // Find the last opening paren that starts the column list + // For: CREATE INDEX ... ON tablename (expr1, expr2) + // or: CREATE INDEX ... ON tablename USING btree (expr1, expr2) + const onMatch = colSection.match(/\bON\b\s+\S+\s+(?:USING \w+\s+)?\((.+)\)\s*$/); + if (!onMatch) { + return { + key: {}, + unique, + ...(sparse ? { sparse: true } : {}) + }; + } + + const exprList = onMatch[1]; + const key = {}; + let type; + + // Split on commas that are not inside parentheses + const exprs = splitExpressions(exprList); + + for (const expr of exprs) { + const trimmed = expr.trim(); + + // Check for _id column + if (/^_id\b/.test(trimmed)) { + const direction = /\bDESC\b/.test(trimmed) ? -1 : 1; + key._id = direction; + continue; + } + + // Numeric type: ((data->>'field')::numeric) or nested variant + const numericMatch = trimmed.match(/::numeric/); + if (numericMatch) { + type = 'number'; + const fieldName = jsonPathToFieldName(trimmed); + if (fieldName) { + const direction = /\bDESC\b/.test(trimmed) ? -1 : 1; + key[fieldName] = direction; + } + continue; + } + + // Date type: data->'field'->>'$date' (or PostgreSQL normalized: + // (data -> 'createdAt'::text) ->> '$date'::text) + const dateMatch = trimmed.match(/->>?\s*'\$date'/); + if (dateMatch) { + type = 'date'; + // The jsonPathToFieldName helper already skips $date segments, + // so we can pass the whole expression + const fieldName = jsonPathToFieldName(trimmed); + if (fieldName) { + const direction = /\bDESC\b/.test(trimmed) ? -1 : 1; + key[fieldName] = direction; + } + continue; + } + + // Default text type: data->>'field' or data->'a'->>'b' + const fieldName = jsonPathToFieldName(trimmed); + if (fieldName) { + const direction = /\bDESC\b/.test(trimmed) ? -1 : 1; + key[fieldName] = direction; + } + } + + return { + key, + unique, + ...(sparse ? { sparse: true } : {}), + ...(type ? { type } : {}) + }; +} + +// Convert a JSONB path expression like data->>'slug' or data->'user'->>'name' +// back into a dot-separated field name like 'slug' or 'user.name'. +// Handles PostgreSQL's normalized output which adds spaces, ::text casts, +// and extra parentheses (e.g. ((data ->> 'a'::text))). +function jsonPathToFieldName(expr) { + // Remove outer parens, ::text and ::numeric casts, ASC/DESC + let cleaned = expr + .replace(/\(+/g, '') + .replace(/\)+/g, '') + .replace(/::(?:text|numeric)/g, '') + .replace(/\s+(ASC|DESC)\s*$/i, '') + .trim(); + + // Must start with 'data' + if (!cleaned.startsWith('data')) { + return null; + } + // Remove the 'data' prefix + cleaned = cleaned.substring(4); + + const parts = []; + // Match ->> 'name' or -> 'name' segments (with optional spaces around arrows) + const segmentPattern = /\s*->>\s*'([^']*)'\s*|\s*->\s*'([^']*)'\s*/g; + let m; + while ((m = segmentPattern.exec(cleaned)) !== null) { + // ->> captures in group 1, -> captures in group 2 + const name = m[1] !== undefined ? m[1] : m[2]; + // Skip the $date pseudo-field used for date indexes + if (name === '$date') { + continue; + } + parts.push(name); + } + + return parts.length > 0 ? parts.join('.') : null; +} + +// Split a comma-separated expression list, respecting parentheses nesting. +// E.g. "((data->>'a')::numeric) DESC, (data->>'b') ASC" → two expressions. +function splitExpressions(str) { + const results = []; + let depth = 0; + let current = ''; + for (const ch of str) { + if (ch === '(') { + depth++; + } else if (ch === ')') { + depth--; + } + if (ch === ',' && depth === 0) { + results.push(current); + current = ''; + } else { + current += ch; + } + } + if (current.trim()) { + results.push(current); + } + return results; +} + +// ============================================================================= +// Query Building (with validated inputs only) +// +// IMPORTANT: buildWhereClause and buildOperatorClause MUTATE the `params` array +// by pushing values onto it. The returned SQL string contains positional +// placeholders ($1, $2, etc.) that reference these array indices. The current +// length of `params` determines the next placeholder number. +// +// Correct usage: +// const params = []; +// const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); +// // params is now populated, whereClause contains matching $N references +// await pool.query(`SELECT * FROM t WHERE ${whereClause}`, params); +// +// DO NOT reuse a params array across independent queries. +// ============================================================================= + +// Build a single JSON arrow step for a path part. +// Numeric parts (array indices) use -> N, string parts use -> 'name'. +function jsonArrow(part) { + return /^\d+$/.test(part) ? `->${part}` : `->'${escapeString(part)}'`; +} + +// Build JSON path for nested fields (returns jsonb) +function buildJsonPath(field, prefix = 'data') { + const parts = field.split('.'); + let path = prefix; + for (const part of parts) { + path += jsonArrow(part); + } + return path; +} + +// Build JSON text path for nested fields (returns text, not jsonb). +// +// The result is wrapped with `COLLATE "C"` so string comparisons and sorts +// follow byte-wise Unicode code point order, matching MongoDB's default +// (non-collated) string semantics. Without this, Postgres would use the +// database/OS locale (typically en_US.UTF-8) which produces user-visible +// divergences from MongoDB in $gt/$lt/sort/range queries and also defeats +// btree indexes built on the same expression whenever predicates and +// indexes disagree on collation. +function buildJsonTextPath(field, prefix = 'data') { + const parts = field.split('.'); + let path = prefix; + for (let i = 0; i < parts.length - 1; i++) { + path += jsonArrow(parts[i]); + } + // Last segment uses ->> for text extraction + const last = parts[parts.length - 1]; + path += /^\d+$/.test(last) ? `->>${last}` : `->>'${escapeString(last)}'`; + return `(${path}) COLLATE "C"`; +} + +/** + * Convert a MongoDB query object to a PostgreSQL WHERE clause. + * + * MUTATES `params` by pushing values for parameterized query placeholders. + * The returned SQL references these values as $1, $2, etc., based on their + * position in `params` at the time each value is added. + * + * @param {Object} query - MongoDB-style query object + * @param {Array} params - Array to append parameter values to (MUTATED) + * @param {string} [prefix='data'] - Column name prefix for JSONB access + * @returns {string} SQL WHERE clause (without "WHERE" keyword) + */ +// Build a SQL condition matching `textExpr` against `regex`. +// +// When the regex is anchored and begins with a literal prefix, emit +// `textExpr >= P AND textExpr < upper(P) AND textExpr ~ src`. The range +// predicate is btree-indexable; the residual regex preserves correctness +// for any trailing pattern. MUTATES `params`. +function buildRegexMatchSql(textExpr, regex, params) { + const flags = regex.ignoreCase ? '*' : ''; + const { prefix } = extractAnchoredLiteralPrefix(regex); + const parts = []; + if (prefix) { + params.push(prefix); + parts.push(`${textExpr} >= $${params.length}`); + const upper = prefixUpperBound(prefix); + if (upper !== null) { + params.push(upper); + parts.push(`${textExpr} < $${params.length}`); + } + } + params.push(regex.source); + parts.push(`${textExpr} ~${flags} $${params.length}`); + return parts.length > 1 ? `(${parts.join(' AND ')})` : parts[0]; +} + +function buildWhereClause(query, params, prefix = 'data', options = {}) { + const _pStart = profileStart(); + const conditions = []; + + for (const [ key, value ] of Object.entries(query || {})) { + if (key === '$and') { + if (!Array.isArray(value)) { + throw new Error('$and must be an array'); + } + const andConditions = value.map(subQuery => { + const subClause = buildWhereClause(subQuery, params, prefix, options); + return `(${subClause})`; + }); + conditions.push(`(${andConditions.join(' AND ')})`); + } else if (key === '$or') { + if (!Array.isArray(value)) { + throw new Error('$or must be an array'); + } + const orConditions = value.map(subQuery => { + const subClause = buildWhereClause(subQuery, params, prefix, options); + return `(${subClause})`; + }); + conditions.push(`(${orConditions.join(' OR ')})`); + } else if (key === '$text') { + // Full-text search: { $text: { $search: "term" } } + // Search against the text index fields stored in the table. + // We use PostgreSQL's to_tsvector/to_tsquery for this. + const searchTerm = value.$search; + if (typeof searchTerm !== 'string') { + throw new Error('$text.$search must be a string'); + } + const textFields = options.textFields || [ + 'highSearchText', 'lowSearchText', 'title', 'searchBoost' + ]; + const tsvectorExpr = buildTsvectorExpr(textFields); + const tsqueryExpr = buildTsqueryParam(searchTerm, params); + if (!tsqueryExpr) { + conditions.push('FALSE'); + } else { + conditions.push(`${tsvectorExpr} @@ ${tsqueryExpr}`); + } + } else if (key === '_id') { + // _id is a separate column, handle specially + if (value instanceof RegExp) { + conditions.push(buildRegexMatchSql('_id', value, params)); + } else if (typeof value === 'object' && value !== null && !Array.isArray(value)) { + conditions.push(buildOperatorClause('_id', value, params, true)); + } else { + params.push(value); + conditions.push(`_id = $${params.length}`); + } + } else if (key.startsWith('$')) { + throw new Error(`Unsupported top-level operator: ${key}`); + } else if (typeof value === 'object' && value !== null && !Array.isArray(value) && !(value instanceof Date) && !(value instanceof RegExp)) { + // Check if it's an operator object + const keys = Object.keys(value); + if (keys.some(k => k.startsWith('$'))) { + conditions.push(buildOperatorClause(key, value, params, false)); + } else { + // Nested object equality + params.push(JSON.stringify(value)); + conditions.push(`${prefix}->>'${escapeString(key)}' = $${params.length}`); + } + } else if (value instanceof RegExp) { + const jsonPath = buildJsonPath(key, prefix); + const textExpr = buildJsonTextPath(key, prefix); + // Scalar match uses the indexable range + residual regex rewrite. + const scalarMatch = buildRegexMatchSql(textExpr, value, params); + // Array-element fallback (regex only — no per-element index to exploit). + const flags = value.ignoreCase ? '*' : ''; + params.push(value.source); + const arrayMatch = `(jsonb_typeof(${jsonPath}) = 'array' AND EXISTS(SELECT 1 FROM jsonb_array_elements_text(${jsonPath}) elem WHERE elem ~${flags} $${params.length}))`; + conditions.push(`(${scalarMatch} OR ${arrayMatch})`); + } else if (value === null || value === undefined) { + // MongoDB: { field: null } and { field: undefined } both match + // explicit null AND missing field + const jsonPath = buildJsonPath(key, prefix); + conditions.push(`(${jsonPath} IS NULL OR ${jsonPath} = 'null'::jsonb)`); + } else { + // Simple equality: use per-field @> which handles both scalar + // equality AND array-contains-scalar in one operation, replacing + // the previous two-branch OR with a single containment check. + const jsonPath = buildJsonPath(key, prefix); + const serialized = serializeValue(value); + params.push(JSON.stringify(serialized)); + conditions.push(`${jsonPath} @> $${params.length}::jsonb`); + } + } + + profileEnd('buildWhereClause', _pStart); + return conditions.length > 0 ? conditions.join(' AND ') : 'TRUE'; +} + +/** + * Build SQL conditions for MongoDB query operators ($eq, $gt, $in, etc.). + * + * MUTATES `params` by pushing values for parameterized query placeholders. + * Called by buildWhereClause for fields with operator objects. + * + * @param {string} field - Field name to apply operators to + * @param {Object} operators - Object with MongoDB operators (e.g., { $gt: 5, $lt: 10 }) + * @param {Array} params - Array to append parameter values to (MUTATED) + * @param {boolean} [isIdField=false] - True if operating on _id column directly + * @returns {string} SQL condition string (e.g., "field > $1 AND field < $2") + */ +function buildOperatorClause(field, operators, params, isIdField = false) { + const conditions = []; + + const jsonPath = isIdField ? '_id' : buildJsonPath(field); + const jsonTextPath = isIdField ? '_id' : buildJsonTextPath(field); + + for (const [ op, opValue ] of Object.entries(operators)) { + switch (op) { + case '$eq': + if (isIdField) { + if (opValue === null || opValue === undefined) { + conditions.push('_id IS NULL'); + } else { + params.push(opValue); + conditions.push(`_id = $${params.length}`); + } + } else { + if (opValue === null || opValue === undefined) { + // $eq: null matches both missing fields and explicit null values + conditions.push(`(${jsonPath} IS NULL OR ${jsonPath} = 'null'::jsonb)`); + } else { + params.push(JSON.stringify(serializeValue(opValue))); + conditions.push(`${jsonPath} = $${params.length}::jsonb`); + } + } + break; + + case '$ne': + if (isIdField) { + if (opValue === null || opValue === undefined) { + conditions.push('_id IS NOT NULL'); + } else { + params.push(opValue); + conditions.push(`(_id IS NULL OR _id != $${params.length})`); + } + } else { + if (opValue === null || opValue === undefined) { + // $ne: null means "field exists and is not null" + // Must exclude both missing fields (IS NULL) and JSON null values + conditions.push(`(${jsonPath} IS NOT NULL AND ${jsonPath} != 'null'::jsonb)`); + } else { + params.push(JSON.stringify(serializeValue(opValue))); + conditions.push(`(${jsonPath} IS NULL OR ${jsonPath} != $${params.length}::jsonb)`); + } + } + break; + + case '$gt': + if (isIdField) { + params.push(opValue); + conditions.push(`_id > $${params.length}`); + } else if (opValue instanceof Date) { + params.push(opValue.toISOString()); + conditions.push(`${jsonPath}->>'$date' > $${params.length}`); + } else if (typeof opValue === 'string') { + params.push(opValue); + conditions.push(`${jsonTextPath} > $${params.length}`); + } else { + params.push(opValue); + conditions.push(`(NULLIF(${jsonTextPath}, ''))::numeric > $${params.length}`); + } + break; + + case '$gte': + if (isIdField) { + params.push(opValue); + conditions.push(`_id >= $${params.length}`); + } else if (opValue instanceof Date) { + params.push(opValue.toISOString()); + conditions.push(`${jsonPath}->>'$date' >= $${params.length}`); + } else if (typeof opValue === 'string') { + params.push(opValue); + conditions.push(`${jsonTextPath} >= $${params.length}`); + } else { + params.push(opValue); + conditions.push(`(NULLIF(${jsonTextPath}, ''))::numeric >= $${params.length}`); + } + break; + + case '$lt': + if (isIdField) { + params.push(opValue); + conditions.push(`_id < $${params.length}`); + } else if (opValue instanceof Date) { + params.push(opValue.toISOString()); + conditions.push(`${jsonPath}->>'$date' < $${params.length}`); + } else if (typeof opValue === 'string') { + params.push(opValue); + conditions.push(`${jsonTextPath} < $${params.length}`); + } else { + params.push(opValue); + conditions.push(`(NULLIF(${jsonTextPath}, ''))::numeric < $${params.length}`); + } + break; + + case '$lte': + if (isIdField) { + params.push(opValue); + conditions.push(`_id <= $${params.length}`); + } else if (opValue instanceof Date) { + params.push(opValue.toISOString()); + conditions.push(`${jsonPath}->>'$date' <= $${params.length}`); + } else if (typeof opValue === 'string') { + params.push(opValue); + conditions.push(`${jsonTextPath} <= $${params.length}`); + } else { + params.push(opValue); + conditions.push(`(NULLIF(${jsonTextPath}, ''))::numeric <= $${params.length}`); + } + break; + + case '$in': + if (!Array.isArray(opValue)) { + throw new Error('$in requires an array'); + } + if (opValue.length === 0) { + // $in with empty array matches nothing + conditions.push('FALSE'); + } else if (isIdField) { + // Single array parameter allows PostgreSQL to cache the plan + // regardless of how many IDs are passed + params.push(opValue); + conditions.push(`_id = ANY($${params.length}::text[])`); + } else { + const hasNull = opValue.includes(null); + const regexValues = opValue.filter(v => v instanceof RegExp); + const nonNullValues = opValue.filter(v => v !== null && !(v instanceof RegExp)); + const parts = []; + if (nonNullValues.length > 0) { + params.push(nonNullValues.map(v => JSON.stringify(serializeValue(v)))); + parts.push(`${jsonPath} @> ANY($${params.length}::jsonb[])`); + } + if (hasNull) { + parts.push(`${jsonPath} IS NULL`); + } + // MongoDB supports RegExp values inside $in for pattern matching + for (const regex of regexValues) { + parts.push(buildRegexMatchSql(jsonTextPath, regex, params)); + } + conditions.push(parts.length > 1 ? `(${parts.join(' OR ')})` : parts[0]); + } + break; + + case '$nin': + if (!Array.isArray(opValue)) { + throw new Error('$nin requires an array'); + } + if (opValue.length === 0) { + // $nin with empty array matches everything + conditions.push('TRUE'); + } else if (isIdField) { + const placeholders = opValue.map(v => { + params.push(v); + return `$${params.length}`; + }); + conditions.push(`(_id IS NULL OR _id NOT IN (${placeholders.join(', ')}))`); + } else { + const hasNull = opValue.includes(null); + const nonNullValues = opValue.filter(v => v !== null); + const parts = []; + if (nonNullValues.length > 0) { + params.push(nonNullValues.map(v => JSON.stringify(serializeValue(v)))); + parts.push(`NOT ${jsonPath} @> ANY($${params.length}::jsonb[])`); + } + if (hasNull) { + // $nin with null means exclude docs where field is null/missing + parts.push(`${jsonPath} IS NOT NULL`); + } else { + // When null is NOT in $nin, still allow null/missing fields through + parts.push(`${jsonPath} IS NULL`); + } + conditions.push(`(${parts.join(hasNull ? ' AND ' : ' OR ')})`); + } + break; + + case '$exists': + if (isIdField) { + conditions.push(opValue ? '_id IS NOT NULL' : '_id IS NULL'); + } else { + conditions.push(opValue ? `${jsonPath} IS NOT NULL` : `${jsonPath} IS NULL`); + } + break; + + case '$not': { + if (typeof opValue !== 'object' || opValue === null) { + throw new Error('$not requires an object'); + } + const negatedClause = buildOperatorClause( + field, opValue, params, isIdField + ); + conditions.push(`NOT (${negatedClause})`); + break; + } + + case '$regex': { + const pattern = opValue instanceof RegExp + ? opValue.source + : String(opValue); + const regexOptions = operators.$options || ''; + const caseInsensitive = regexOptions.includes('i'); + // Reconstruct a RegExp so the helper can analyze the pattern uniformly. + let regex; + try { + regex = new RegExp(pattern, caseInsensitive ? 'i' : ''); + } catch (e) { + // Fall back to direct emission if the pattern isn't a valid JS RegExp + params.push(pattern); + conditions.push( + `${isIdField ? '_id' : jsonTextPath} ~${caseInsensitive ? '*' : ''} $${params.length}` + ); + break; + } + conditions.push( + buildRegexMatchSql(isIdField ? '_id' : jsonTextPath, regex, params) + ); + break; + } + + case '$options': + // Handled with $regex, skip + break; + + case '$all': + if (!Array.isArray(opValue)) { + throw new Error('$all requires an array'); + } + params.push(JSON.stringify(opValue)); + conditions.push(`${jsonPath} @> $${params.length}::jsonb`); + break; + + case '$size': + params.push(opValue); + conditions.push( + `jsonb_typeof(${jsonPath}) = 'array' AND jsonb_array_length(${jsonPath}) = $${params.length}` + ); + break; + + default: + throw new Error(`Unsupported operator: ${op}`); + } + } + + return conditions.join(' AND '); +} + +// Build the tsvector SQL expression for a given list of text fields. +// Used by both the WHERE clause ($text matching) and SELECT (ts_rank scoring). +function buildTsvectorExpr(textFields) { + const parts = textFields.map(f => { + const fieldParts = f.split('.'); + let path = 'data'; + for (let i = 0; i < fieldParts.length - 1; i++) { + path += `->'${escapeString(fieldParts[i])}'`; + } + path += `->>'${escapeString(fieldParts[fieldParts.length - 1])}'`; + // COLLATE "C" here is ignored for the tsvector output but keeps this + // expression textually identical to the CREATE INDEX expression + // produced by createIndex so the planner can match the gin index. + return `coalesce((${path}) COLLATE "C", '')`; + }); + return `to_tsvector('simple', ${parts.join(' || \' \' || ')})`; +} + +// Build the tsquery SQL expression from a search string. +// Returns { expr, params } where expr contains $N placeholders. +function buildTsqueryParam(searchTerm, params) { + const words = searchTerm.trim().split(/\s+/).filter(w => w.length > 0); + if (words.length === 0) { + return null; + } + params.push(words.map(w => w.replace(/[&|!():*<>'"]/g, ' ')).join(' | ')); + return `to_tsquery('simple', $${params.length})`; +} + +// Check if a query object contains a $text operator with a non-empty search string. +function queryHasText(query) { + if (!query || typeof query !== 'object' || !('$text' in query)) { + return false; + } + const search = query.$text && query.$text.$search; + if (typeof search !== 'string') { + return false; + } + return search.trim().split(/\s+/).filter(w => w.length > 0).length > 0; +} + +// Build ORDER BY clause +function buildOrderBy(sort, options = {}) { + const _pStart = profileStart(); + const clauses = []; + + if (sort && Object.keys(sort).length > 0) { + for (const [ field, direction ] of Object.entries(sort)) { + if (direction && typeof direction === 'object' && direction.$meta === 'textScore') { + // Sort by text search relevance score (descending — higher is better) + if (options.hasTextScore) { + clauses.push('_score DESC'); + } + continue; + } + if (field === '_id') { + clauses.push(`_id ${direction === -1 ? 'DESC' : 'ASC'}`); + } else { + const jsonPath = buildJsonTextPath(field); + clauses.push(`${jsonPath} ${direction === -1 ? 'DESC' : 'ASC'}`); + } + } + } + + // Always add _order as final tiebreaker to match MongoDB's + // insertion-order stability among equal sort keys + clauses.push('_order ASC'); + profileEnd('buildOrderBy', _pStart); + return `ORDER BY ${clauses.join(', ')}`; +} + +// ============================================================================= +// Cursor Implementation +// ============================================================================= + +class PostgresCursor { + constructor(collection, query, options = {}) { + this._collection = collection; + this._query = query; + this._projection = options.projection || null; + this._sort = null; + this._limit = null; + this._skip = null; + this._cursorClient = null; + this._cursorName = null; + this._exhausted = false; + } + + project(projection) { + this._projection = projection; + return this; + } + + sort(sort) { + this._sort = sort; + return this; + } + + limit(n) { + // MongoDB convention: limit(0) means no limit + const val = validateInteger(n, 'limit'); + this._limit = val === 0 ? null : val; + return this; + } + + skip(n) { + this._skip = validateInteger(n, 'skip'); + return this; + } + + clone() { + const cloned = new PostgresCursor(this._collection, this._query); + cloned._projection = this._projection; + cloned._sort = this._sort; + cloned._limit = this._limit; + cloned._skip = this._skip; + return cloned; + } + + // Build the SELECT SQL + params that this cursor would execute. Shared + // by toArray/_next and exposed via explain() so tests and callers can + // introspect the planned query without re-deriving the SQL by hand. + _buildFindSql() { + const params = []; + const queryOptions = this._collection._queryOptions(); + const qualifiedName = this._collection._qualifiedName(); + const whereClause = buildWhereClause(this._query, params, 'data', queryOptions); + const hasText = queryHasText(this._query); + const orderBy = buildOrderBy(this._sort, { hasTextScore: hasText }); + + // When a $text query is active, compute ts_rank as _score + let selectCols = '_id, data'; + if (hasText) { + const textFields = queryOptions.textFields || [ + 'highSearchText', 'lowSearchText', 'title', 'searchBoost' + ]; + const tsvectorExpr = buildTsvectorExpr(textFields); + const tsqueryExpr = buildTsqueryParam(this._query.$text.$search, params); + if (tsqueryExpr) { + selectCols += `, ts_rank(${tsvectorExpr}, ${tsqueryExpr}) AS _score`; + } + } + + let sql = `SELECT ${selectCols} FROM ${qualifiedName} WHERE ${whereClause} ${orderBy}`; + if (this._limit != null) { + sql += ` LIMIT ${this._limit}`; + } + if (this._skip != null) { + sql += ` OFFSET ${this._skip}`; + } + return { + sql, + params + }; + } + + // Returns the SQL and parameter values the adapter would execute for + // this cursor's current query/sort/limit/skip/projection. Useful for + // EXPLAIN-based tests and for debugging query planner behavior. The + // returned SQL uses the adapter's native placeholder style ($N for + // PostgreSQL). + async explain() { + await this._collection._ensureTable(); + return this._buildFindSql(); + } + + async toArray() { + const _pStart = profileStart(); + await this._collection._ensureTable(); + + const { sql, params } = this._buildFindSql(); + + const _qStart = profileStart(); + const result = await this._collection._pool.query(sql, params); + profileQuery(sql, _qStart); + const rows = result.rows.map(row => { + const doc = deserializeDocument(row.data, row._id); + const meta = row._score != null ? { textScore: parseFloat(row._score) } : {}; + return this._projection ? applyProjection(doc, this._projection, meta) : doc; + }); + profileEnd('findToArray', _pStart); + return rows; + } + + next(callback) { + const promise = this._next(); + if (callback) { + promise.then(doc => callback(null, doc), err => callback(err)); + return; + } + return promise; + } + + async _next() { + if (this._peeked !== undefined) { + const doc = this._peeked; + this._peeked = undefined; + if (doc === null) { + this._exhausted = true; + } + return doc; + } + if (this._exhausted) { + return null; + } + if (!this._cursorClient) { + await this._collection._ensureTable(); + this._cursorClient = await this._collection._pool.connect(); + this._cursorName = `cur_${generateId()}`; + + const { sql, params } = this._buildFindSql(); + + const escapedCursorName = escapeIdentifier(this._cursorName); + await this._cursorClient.query('BEGIN'); + await this._cursorClient.query( + `DECLARE "${escapedCursorName}" CURSOR FOR ${sql}`, + params + ); + } + + const escapedCursorName = escapeIdentifier(this._cursorName); + const result = await this._cursorClient.query( + `FETCH NEXT FROM "${escapedCursorName}"` + ); + + if (result.rows.length === 0) { + this._exhausted = true; + await this._cursorClient.query('COMMIT'); + this._cursorClient.release(); + this._cursorClient = null; + return null; + } + + const row = result.rows[0]; + const doc = deserializeDocument(row.data, row._id); + const meta = row._score != null ? { textScore: parseFloat(row._score) } : {}; + return this._projection ? applyProjection(doc, this._projection, meta) : doc; + } + + async hasNext() { + if (this._exhausted) { + return false; + } + if (this._peeked !== undefined) { + return this._peeked !== null; + } + this._peeked = await this._next(); + return this._peeked !== null; + } + + async close() { + if (this._cursorClient) { + const escapedCursorName = escapeIdentifier(this._cursorName); + await this._cursorClient.query(`CLOSE "${escapedCursorName}"`); + await this._cursorClient.query('COMMIT'); + this._cursorClient.release(); + this._cursorClient = null; + this._exhausted = true; + } + } + + addCursorFlag() { + // No-op for PostgreSQL — flags like noCursorTimeout are MongoDB-specific + return this; + } + + [Symbol.asyncIterator]() { + return { + cursor: this, + async next() { + const doc = await this.cursor._next(); + if (doc === null) { + return { + done: true, + value: undefined + }; + } + return { + done: false, + value: doc + }; + } + }; + } + + async count() { + const _pStart = profileStart(); + await this._collection._ensureTable(); + + const params = []; + const qualifiedName = this._collection._qualifiedName(); + const whereClause = buildWhereClause(this._query, params, 'data', this._collection._queryOptions()); + const sql = `SELECT COUNT(*) as count FROM ${qualifiedName} WHERE ${whereClause}`; + const _qStart = profileStart(); + const result = await this._collection._pool.query(sql, params); + profileQuery(sql, _qStart); + profileEnd('countDocuments', _pStart); + return parseInt(result.rows[0].count, 10); + } +} + +// ============================================================================= +// Collection Implementation +// ============================================================================= + +class PostgresCollection { + constructor(db, name) { + this._db = db; + this._pool = db._pool; + this._tableName = validateTableName(name); + this._schema = db._schema || null; + this._name = name; + this._indexes = new Map(); + this._textFields = null; + this._initialized = false; + } + + // Returns the schema-qualified table name for use in SQL. + // In multi-schema mode: "schemaname"."tablename" + // In simple mode: "tablename" + _qualifiedName() { + const table = `"${escapeIdentifier(this._tableName)}"`; + if (this._schema) { + return `"${escapeIdentifier(this._schema)}".${table}`; + } + return table; + } + + _queryOptions() { + return this._textFields ? { textFields: this._textFields } : {}; + } + + get collectionName() { + return this._name; + } + + get name() { + return this._name; + } + + async _ensureTable() { + if (this._initialized) { + return; + } + const _pStart = profileStart(); + + // In multi-schema mode, ensure the schema exists + if (this._schema) { + await this._pool.query( + `CREATE SCHEMA IF NOT EXISTS "${escapeIdentifier(this._schema)}"` + ); + } + + const qualifiedName = this._qualifiedName(); + await this._pool.query(` + CREATE TABLE IF NOT EXISTS ${qualifiedName} ( + _id TEXT COLLATE "C" PRIMARY KEY, + _order SERIAL, + data JSONB NOT NULL + ) + `); + // Add _order column to tables created before it existed + try { + await this._pool.query(` + ALTER TABLE ${qualifiedName} ADD COLUMN IF NOT EXISTS _order SERIAL + `); + } catch (e) { + // Column already exists, ignore + } + profileEnd('ensureTable', _pStart); + this._initialized = true; + } + + async insertOne(doc) { + await this._ensureTable(); + + const id = doc._id != null ? String(doc._id) : generateId(); + const docWithoutId = { ...doc }; + delete docWithoutId._id; + + const qualifiedName = this._qualifiedName(); + try { + await this._pool.query( + `INSERT INTO ${qualifiedName} (_id, data) VALUES ($1, $2)`, + [ id, serializeDocument(docWithoutId) ] + ); + return { + acknowledged: true, + insertedId: id, + insertedCount: 1, + ops: [ { + ...doc, + _id: id + } ], + result: { ok: 1 } + }; + } catch (e) { + if (e.code === '23505') { + throw makeDuplicateKeyError(e); + } + throw e; + } + } + + async insertMany(docs) { + await this._ensureTable(); + + const insertedIds = {}; + let insertedCount = 0; + + for (let i = 0; i < docs.length; i++) { + const result = await this.insertOne(docs[i]); + insertedIds[i] = result.insertedId; + insertedCount++; + } + + return { + acknowledged: true, + insertedCount, + insertedIds, + result: { ok: 1 } + }; + } + + async findOne(query, options = {}) { + const _pStart = profileStart(); + await this._ensureTable(); + + const params = []; + const qualifiedName = this._qualifiedName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + const sql = `SELECT _id, data FROM ${qualifiedName} WHERE ${whereClause} LIMIT 1`; + + const _qStart = profileStart(); + const result = await this._pool.query(sql, params); + profileQuery(sql, _qStart); + if (result.rows.length === 0) { + profileEnd('findOne', _pStart); + return null; + } + + const doc = deserializeDocument(result.rows[0].data, result.rows[0]._id); + const final = options.projection ? applyProjection(doc, options.projection) : doc; + profileEnd('findOne', _pStart); + return final; + } + + find(query, options) { + return new PostgresCursor(this, query, options); + } + + async updateOne(query, update, options = {}) { + const _pStart = profileStart(); + await this._ensureTable(); + + // Handle legacy callback as third argument (ignore the callback, PostgreSQL + // adapter is Promise-based) + if (typeof options === 'function') { + options = {}; + } + + // Single-statement fast path: when the update uses only simple + // operators without upsert, execute a single UPDATE statement + // instead of the read-modify-write cycle. $push, $pull and + // $addToSet are included only when all their values are scalars. + if (!options.upsert) { + const atomicOps = [ + '$inc', '$set', '$unset', '$currentDate', + '$push', '$pull', '$addToSet' + ]; + const ops = Object.keys(update); + const isAtomicCompatible = ops.length > 0 && + ops.every(op => atomicOps.includes(op)); + if (isAtomicCompatible) { + const allScalar = [ '$push', '$pull', '$addToSet' ].every(op => { + if (!update[op]) { + return true; + } + return Object.values(update[op]).every(v => + typeof v === 'string' || + typeof v === 'number' || + typeof v === 'boolean' + ); + }); + if (allScalar) { + const result = await this._atomicUpdateOne(query, update); + profileEnd('updateOne', _pStart); + return result; + } + } + } + + const params = []; + const qualifiedName = this._qualifiedName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + const selectSql = `SELECT _id, data FROM ${qualifiedName} WHERE ${whereClause} LIMIT 1`; + const _qStart = profileStart(); + const selectResult = await this._pool.query(selectSql, params); + profileQuery(selectSql, _qStart); + + if (selectResult.rows.length === 0) { + if (options.upsert) { + let newDoc = {}; + if (query._id) { + newDoc._id = query._id; + } + newDoc = applyUpdate(newDoc, update); + const insertResult = await this.insertOne(newDoc); + return { + acknowledged: true, + matchedCount: 0, + modifiedCount: 0, + upsertedId: insertResult.insertedId, + upsertedCount: 1, + result: { + nModified: 0, + n: 1 + } + }; + } + return { + acknowledged: true, + matchedCount: 0, + modifiedCount: 0, + result: { + nModified: 0, + n: 0 + } + }; + } + + const row = selectResult.rows[0]; + const existing = deserializeDocument(row.data, row._id); + const updated = applyUpdate(existing, update); + const { _id, ...dataWithoutId } = updated; + + try { + await this._pool.query( + `UPDATE ${qualifiedName} SET data = $1 WHERE _id = $2`, + [ serializeDocument(dataWithoutId), selectResult.rows[0]._id ] + ); + } catch (e) { + if (e.code === '23505') { + throw makeDuplicateKeyError(e); + } + throw e; + } + + return { + acknowledged: true, + matchedCount: 1, + modifiedCount: 1, + result: { + nModified: 1, + n: 1 + } + }; + } + + // Atomic update using SQL expressions for $inc, $set, $unset, + // $currentDate, $push, $pull, $addToSet (no read-modify-write race) + async _atomicUpdateOne(query, update) { + const params = []; + const qualifiedName = this._qualifiedName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + // Build a chain of jsonb_set / #- calls for atomic update + let dataExpr = 'data'; + + if (update.$set) { + for (const [ field, value ] of Object.entries(update.$set)) { + const pathArray = field.split('.'); + const pathLiteral = `'{${pathArray.map(p => escapeString(p)).join(',')}}'`; + const serialized = serializeValue(value); + params.push(JSON.stringify(serialized)); + dataExpr = `jsonb_set(${dataExpr}, ${pathLiteral}, $${params.length}::jsonb, true)`; + } + } + + if (update.$inc) { + for (const [ field, value ] of Object.entries(update.$inc)) { + const pathArray = field.split('.'); + const pathLiteral = `'{${pathArray.map(p => escapeString(p)).join(',')}}'`; + let readPath = 'data'; + for (let i = 0; i < pathArray.length - 1; i++) { + readPath += `->'${escapeString(pathArray[i])}'`; + } + readPath += `->>'${escapeString(pathArray[pathArray.length - 1])}'`; + params.push(value); + dataExpr = `jsonb_set(${dataExpr}, ${pathLiteral}, to_jsonb(COALESCE((${readPath})::numeric, 0) + $${params.length}), true)`; + } + } + + if (update.$unset) { + const fields = Array.isArray(update.$unset) + ? update.$unset + : Object.keys(update.$unset); + for (const field of fields) { + const pathArray = field.split('.'); + const pathLiteral = `'{${pathArray.map(p => escapeString(p)).join(',')}}'`; + dataExpr = `(${dataExpr}) #- ${pathLiteral}`; + } + } + + if (update.$currentDate) { + for (const [ field, value ] of Object.entries(update.$currentDate)) { + if (value === true || (value && value.$type === 'date')) { + const pathArray = field.split('.'); + const pathLiteral = `'{${pathArray.map(p => escapeString(p)).join(',')}}'`; + const dateVal = JSON.stringify(serializeValue(new Date())); + params.push(dateVal); + dataExpr = `jsonb_set(${dataExpr}, ${pathLiteral}, $${params.length}::jsonb, true)`; + } + } + } + + // $push: append scalar value to array + if (update.$push) { + for (const [ field, value ] of Object.entries(update.$push)) { + const pathArray = field.split('.'); + const pathLiteral = `'{${pathArray.map(p => escapeString(p)).join(',')}}'`; + const readPath = pathArray.map(p => `'${escapeString(p)}'`).join('->'); + const coalesced = `COALESCE(data->${readPath}, '[]'::jsonb)`; + params.push(JSON.stringify(value)); + dataExpr = `jsonb_set(${dataExpr}, ${pathLiteral}, ${coalesced} || $${params.length}::jsonb, true)`; + } + } + + // $pull: remove scalar value from array + if (update.$pull) { + for (const [ field, value ] of Object.entries(update.$pull)) { + const pathArray = field.split('.'); + const pathLiteral = `'{${pathArray.map(p => escapeString(p)).join(',')}}'`; + const readPath = pathArray.map(p => `'${escapeString(p)}'`).join('->'); + params.push(JSON.stringify(value)); + dataExpr = `jsonb_set(${dataExpr}, ${pathLiteral}, ` + + '(SELECT COALESCE(jsonb_agg(elem), \'[]\'::jsonb) ' + + `FROM jsonb_array_elements(COALESCE(data->${readPath}, '[]'::jsonb)) AS elem ` + + `WHERE elem != $${params.length}::jsonb), true)`; + } + } + + // $addToSet: add scalar value to array if not already present + if (update.$addToSet) { + for (const [ field, value ] of Object.entries(update.$addToSet)) { + const pathArray = field.split('.'); + const pathLiteral = `'{${pathArray.map(p => escapeString(p)).join(',')}}'`; + const readPath = pathArray.map(p => `'${escapeString(p)}'`).join('->'); + const coalesced = `COALESCE(data->${readPath}, '[]'::jsonb)`; + params.push(JSON.stringify(value)); + dataExpr = `jsonb_set(${dataExpr}, ${pathLiteral}, ` + + `CASE WHEN ${coalesced} @> $${params.length}::jsonb ` + + `THEN ${coalesced} ` + + `ELSE ${coalesced} || $${params.length}::jsonb END, true)`; + } + } + + const sql = `UPDATE ${qualifiedName} SET data = ${dataExpr} WHERE ${whereClause}`; + try { + const _qStart = profileStart(); + const result = await this._pool.query(sql, params); + profileQuery(sql, _qStart); + const matched = result.rowCount > 0 ? 1 : 0; + return { + acknowledged: true, + matchedCount: matched, + modifiedCount: matched, + result: { + nModified: matched, + n: matched + } + }; + } catch (e) { + if (e.code === '23505') { + throw makeDuplicateKeyError(e); + } + throw e; + } + } + + async updateMany(query, update, options = {}) { + await this._ensureTable(); + + const params = []; + const qualifiedName = this._qualifiedName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + const selectSql = `SELECT _id, data FROM ${qualifiedName} WHERE ${whereClause}`; + const selectResult = await this._pool.query(selectSql, params); + + if (selectResult.rows.length === 0) { + return { + acknowledged: true, + matchedCount: 0, + modifiedCount: 0, + result: { + nModified: 0, + n: 0 + } + }; + } + + let modifiedCount = 0; + for (const row of selectResult.rows) { + const existing = deserializeDocument(row.data, row._id); + const updated = applyUpdate(existing, update); + const { _id, ...dataWithoutId } = updated; + + await this._pool.query( + `UPDATE ${qualifiedName} SET data = $1 WHERE _id = $2`, + [ serializeDocument(dataWithoutId), row._id ] + ); + modifiedCount++; + } + + return { + acknowledged: true, + matchedCount: selectResult.rows.length, + modifiedCount, + result: { + nModified: modifiedCount, + n: selectResult.rows.length + } + }; + } + + async replaceOne(query, replacement, options = {}) { + await this._ensureTable(); + + const params = []; + const qualifiedName = this._qualifiedName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + const selectSql = `SELECT _id FROM ${qualifiedName} WHERE ${whereClause} LIMIT 1`; + const selectResult = await this._pool.query(selectSql, params); + + if (selectResult.rows.length === 0) { + if (options.upsert) { + const result = await this.insertOne(replacement); + return { + acknowledged: true, + matchedCount: 0, + modifiedCount: 0, + upsertedId: result.insertedId, + upsertedCount: 1 + }; + } + return { + acknowledged: true, + matchedCount: 0, + modifiedCount: 0 + }; + } + + const { _id, ...dataWithoutId } = replacement; + try { + await this._pool.query( + `UPDATE ${qualifiedName} SET data = $1 WHERE _id = $2`, + [ serializeDocument(dataWithoutId), selectResult.rows[0]._id ] + ); + } catch (e) { + if (e.code === '23505') { + throw makeDuplicateKeyError(e); + } + throw e; + } + + return { + acknowledged: true, + matchedCount: 1, + modifiedCount: 1 + }; + } + + async deleteOne(query) { + await this._ensureTable(); + + const params = []; + const qualifiedName = this._qualifiedName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + const result = await this._pool.query( + `DELETE FROM ${qualifiedName} WHERE _id IN ( + SELECT _id FROM ${qualifiedName} WHERE ${whereClause} LIMIT 1 + )`, + params + ); + + return { + acknowledged: true, + deletedCount: result.rowCount, + result: { ok: 1 } + }; + } + + async deleteMany(query) { + await this._ensureTable(); + + const params = []; + const qualifiedName = this._qualifiedName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + const result = await this._pool.query( + `DELETE FROM ${qualifiedName} WHERE ${whereClause}`, + params + ); + + return { + acknowledged: true, + deletedCount: result.rowCount, + result: { ok: 1 } + }; + } + + // Legacy MongoDB method aliases used by ApostropheCMS + async remove(query) { + return this.deleteMany(query); + } + + async removeOne(query) { + return this.deleteOne(query); + } + + async removeMany(query) { + return this.deleteMany(query); + } + + async countDocuments(query = {}) { + const _pStart = profileStart(); + await this._ensureTable(); + + const params = []; + const qualifiedName = this._qualifiedName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + const sql = `SELECT COUNT(*) as count FROM ${qualifiedName} WHERE ${whereClause}`; + + const _qStart = profileStart(); + const result = await this._pool.query(sql, params); + profileQuery(sql, _qStart); + profileEnd('countDocuments', _pStart); + return parseInt(result.rows[0].count, 10); + } + + async distinct(field, query = {}) { + const _pStart = profileStart(); + await this._ensureTable(); + + const params = []; + const qualifiedName = this._qualifiedName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + if (field === '_id') { + const sql = `SELECT DISTINCT _id as value FROM ${qualifiedName} WHERE ${whereClause}`; + const _qStart = profileStart(); + const result = await this._pool.query(sql, params); + profileQuery(sql, _qStart); + profileEnd('distinct', _pStart); + return result.rows.map(row => row.value).filter(v => v !== null); + } + + const jsonPath = buildJsonPath(field); + // MongoDB's distinct() automatically flattens arrays. + // Use LATERAL with jsonb_array_elements to unwind array values, + // and fall back to the value itself for scalars. + // Use jsonb_array_elements (not _text) to preserve types for non-string values. + const sql = `SELECT DISTINCT elem as value FROM ${qualifiedName}, LATERAL jsonb_array_elements( + CASE WHEN jsonb_typeof(${jsonPath}) = 'array' THEN ${jsonPath} ELSE jsonb_build_array(${jsonPath}) END + ) AS elem WHERE ${whereClause} AND ${jsonPath} IS NOT NULL`; + const _qStart = profileStart(); + const result = await this._pool.query(sql, params); + profileQuery(sql, _qStart); + + const values = result.rows + .map(row => { + const v = row.value; + if (v === null || v === undefined) { + return null; + } + // pg driver parses jsonb automatically, returning JS types. + // For jsonb strings, we already get JS strings. + // For jsonb numbers/booleans/objects, we get those types. + // No additional parsing needed. + return v; + }) + .filter(v => v !== null); + profileEnd('distinct', _pStart); + return values; + } + + aggregate(pipeline) { + return new AggregationCursor(this, pipeline); + } + + async bulkWrite(operations) { + await this._ensureTable(); + + let insertedCount = 0; + let modifiedCount = 0; + let deletedCount = 0; + let upsertedCount = 0; + const insertedIds = {}; + const upsertedIds = {}; + + for (let i = 0; i < operations.length; i++) { + const op = operations[i]; + + if (op.insertOne) { + const result = await this.insertOne(op.insertOne.document); + insertedIds[i] = result.insertedId; + insertedCount++; + } else if (op.updateOne) { + const result = await this.updateOne( + op.updateOne.filter, + op.updateOne.update, + { upsert: op.updateOne.upsert } + ); + modifiedCount += result.modifiedCount; + if (result.upsertedId) { + upsertedIds[i] = result.upsertedId; + upsertedCount++; + } + } else if (op.updateMany) { + const result = await this.updateMany( + op.updateMany.filter, + op.updateMany.update, + { upsert: op.updateMany.upsert } + ); + modifiedCount += result.modifiedCount; + } else if (op.deleteOne) { + const result = await this.deleteOne(op.deleteOne.filter); + deletedCount += result.deletedCount; + } else if (op.deleteMany) { + const result = await this.deleteMany(op.deleteMany.filter); + deletedCount += result.deletedCount; + } else if (op.replaceOne) { + const result = await this.replaceOne( + op.replaceOne.filter, + op.replaceOne.replacement, + { upsert: op.replaceOne.upsert } + ); + modifiedCount += result.modifiedCount; + if (result.upsertedId) { + upsertedIds[i] = result.upsertedId; + upsertedCount++; + } + } + } + + return { + acknowledged: true, + insertedCount, + modifiedCount, + deletedCount, + upsertedCount, + insertedIds, + upsertedIds + }; + } + + async findOneAndUpdate(query, update, options = {}) { + await this._ensureTable(); + + const params = []; + const qualifiedName = this._qualifiedName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + const selectSql = `SELECT _id, data FROM ${qualifiedName} WHERE ${whereClause} LIMIT 1`; + const selectResult = await this._pool.query(selectSql, params); + + if (selectResult.rows.length === 0) { + if (options.upsert) { + let newDoc = {}; + if (query._id) { + newDoc._id = query._id; + } + newDoc = applyUpdate(newDoc, update); + await this.insertOne(newDoc); + return options.returnDocument === 'after' ? newDoc : null; + } + return null; + } + + const row = selectResult.rows[0]; + const existing = deserializeDocument(row.data, row._id); + const updated = applyUpdate(existing, update); + const { _id, ...dataWithoutId } = updated; + + await this._pool.query( + `UPDATE ${qualifiedName} SET data = $1 WHERE _id = $2`, + [ serializeDocument(dataWithoutId), selectResult.rows[0]._id ] + ); + + return options.returnDocument === 'after' ? updated : existing; + } + + /** + * Create an index on one or more fields. + * + * @param {Object} keys - Fields to index, + * e.g. { fieldName: 1 } or { fieldName: -1 } + * @param {Object} options - Index options + * @param {string} [options.name] - Custom index name + * @param {boolean} [options.unique] - Unique index + * @param {boolean} [options.sparse] - Sparse/partial index + * (only index docs where field exists) + * @param {string} [options.type] - Field type for range + * query optimization: + * - 'number': numeric cast for $gt/$lt on numbers + * - 'date': date extraction for $gt/$lt on dates + * - undefined: text (for $eq, $in, $regex) + * + * IMPORTANT: The `type` option is required for + * PostgreSQL to efficiently use indexes on range + * queries ($gt, $gte, $lt, $lte). MongoDB ignores it. + * + * @example + * // Text/equality index (default) - efficient for $eq, $in, $regex + * await collection.createIndex({ slug: 1 }); + * + * // Numeric index - efficient for $gt, $lt on numbers + * await collection.createIndex({ price: 1 }, { type: 'number' }); + * + * // Date index - for $gt, $lt on dates + * await collection.createIndex( + * { createdAt: 1 }, { type: 'date' } + * ); + * + * // Unique sparse date index + * await collection.createIndex( + * { publishedAt: 1 }, + * { type: 'date', unique: true, sparse: true } + * ); + */ + async createIndex(keys, options = {}) { + await this._ensureTable(); + + const keyEntries = Object.entries(keys); + const indexType = options.type; // 'number', 'date', or undefined (text) + + // Helper to build JSON path expression for index + // (handles nested fields like 'user.name'). + // The type parameter determines the expression + // for range query optimization + const buildIndexPath = (field, type) => { + const parts = field.split('.'); + + if (type === 'date') { + // Dates are stored as { $date: "ISO string" } in UTC + // ISO 8601 strings sort correctly as text (YYYY-MM-DDTHH:MM:SS.sssZ format) + // We index the raw text because timestamp casts aren't IMMUTABLE in PostgreSQL + let path = 'data'; + for (const part of parts) { + path += `->'${escapeString(part)}'`; + } + // Extract $date text value (no cast - text sorts correctly for ISO dates) + return `${path}->>'$date'`; + } + + if (type === 'number') { + // Build path to extract text and cast to numeric + let path = 'data'; + if (parts.length === 1) { + path = `data->>'${escapeString(parts[0])}'`; + } else { + for (let i = 0; i < parts.length - 1; i++) { + path += `->'${escapeString(parts[i])}'`; + } + path += `->>'${escapeString(parts[parts.length - 1])}'`; + } + return `(${path})::numeric`; + } + + // Default: text extraction for equality/text queries. + // + // Wrapped with COLLATE "C" to match buildJsonTextPath so that query + // predicates and this index expression agree on collation; otherwise + // the planner cannot use the index for range/equality comparisons. + // Byte-wise ordering also matches MongoDB's default string semantics. + if (parts.length === 1) { + return `(data->>'${escapeString(parts[0])}') COLLATE "C"`; + } + // For nested: data->'user'->>'name' + let path = 'data'; + for (let i = 0; i < parts.length - 1; i++) { + path += `->'${escapeString(parts[i])}'`; + } + path += `->>'${escapeString(parts[parts.length - 1])}'`; + return `(${path}) COLLATE "C"`; + }; + + // Helper to build JSON path for existence check (returns jsonb, not text) + const buildIndexPathJsonb = (field) => { + const parts = field.split('.'); + let path = 'data'; + for (const part of parts) { + path += `->'${escapeString(part)}'`; + } + return path; + }; + + // Generate a safe index name + const safeFieldNames = keyEntries.map(([ k ]) => k.replace(/[^a-zA-Z0-9]/g, '_')).join('_'); + const indexName = options.name + ? sanitizeIndexName(options.name) + : `idx_${this._tableName}_${safeFieldNames}`.substring(0, 63); + + // Generate MongoDB-compatible index name for indexInformation() compatibility + const mongoName = options.name || keyEntries.map(([ k, v ]) => `${k}_${v}`).join('_'); + + // Store index metadata + this._indexes.set(indexName, { + keys, + options, + mongoName + }); + + const qualifiedName = this._qualifiedName(); + const escapedIndexName = escapeIdentifier(indexName); + + // Build WHERE clause for sparse indexes (PostgreSQL partial index) + let whereClause = ''; + if (options.sparse) { + const sparseConditions = keyEntries.map(([ field ]) => { + if (field === '_id') { + return '_id IS NOT NULL'; + } + return `${buildIndexPathJsonb(field)} IS NOT NULL`; + }); + whereClause = ` WHERE ${sparseConditions.join(' AND ')}`; + } + + // Handle text indexes (full-text search, always uses text extraction) + const hasTextIndex = keyEntries.some(([ , v ]) => v === 'text'); + if (hasTextIndex) { + let textFields = keyEntries.filter(([ , v ]) => v === 'text').map(([ k ]) => k); + // MongoDB dumps store text indexes as { _fts: 'text', _ftsx: 1 } + // The real field names are in options.weights + if (textFields.length === 1 && textFields[0] === '_fts' && options.weights) { + textFields = Object.keys(options.weights); + } + // Store the text index fields so $text queries use them + this._textFields = textFields; + const tsvectorExpr = textFields + .map(f => `coalesce(${buildIndexPath(f, null)}, '')`) + .join(' || \' \' || '); + + await this._pool.query(` + CREATE INDEX IF NOT EXISTS "${escapedIndexName}" + ON ${qualifiedName} + USING gin(to_tsvector('simple', ${tsvectorExpr}))${whereClause} + `); + return indexName; + } + + // Handle unique constraint + if (options.unique) { + const indexExprs = keyEntries.map(([ field ]) => { + return field === '_id' ? '_id' : `(${buildIndexPath(field, indexType)})`; + }); + + await this._pool.query(` + CREATE UNIQUE INDEX IF NOT EXISTS "${escapedIndexName}" + ON ${qualifiedName} (${indexExprs.join(', ')})${whereClause} + `); + return indexName; + } + + // Handle regular indexes (single or compound) + const indexExprs = keyEntries.map(([ field, direction ]) => { + if (field === '_id') { + return `_id ${direction === -1 ? 'DESC' : 'ASC'}`; + } + return `(${buildIndexPath(field, indexType)}) ${direction === -1 ? 'DESC' : 'ASC'}`; + }); + + await this._pool.query(` + CREATE INDEX IF NOT EXISTS "${escapedIndexName}" + ON ${qualifiedName} (${indexExprs.join(', ')})${whereClause} + `); + + return indexName; + } + + async ensureIndex(keys, options) { + return this.createIndex(keys, options); + } + + async dropIndex(indexName) { + // Look up by MongoDB-compatible name first (in case caller uses that) + let pgName = null; + for (const [ pgKey, meta ] of this._indexes.entries()) { + if (meta.mongoName === indexName) { + pgName = pgKey; + break; + } + } + if (!pgName) { + // Try as a direct postgres index name. Use the same sanitizer as + // createIndex so a name like "slug_unique" created via createIndex + // can be dropped with the same string, and MongoDB-style names with + // illegal characters don't throw. + pgName = sanitizeIndexName(indexName); + } + this._indexes.delete(pgName); + const escapedIndexName = escapeIdentifier(pgName); + if (this._schema) { + await this._pool.query( + `DROP INDEX IF EXISTS "${escapeIdentifier(this._schema)}"."${escapedIndexName}"` + ); + } else { + await this._pool.query(`DROP INDEX IF EXISTS "${escapedIndexName}"`); + } + } + + async indexes() { + await this._ensureTable(); + + const schemaName = this._schema || 'public'; + const result = await this._pool.query(` + SELECT indexname, indexdef + FROM pg_indexes + WHERE schemaname = $1 AND tablename = $2 + `, [ schemaName, this._tableName ]); + + const indexes = [ { + name: '_id_', + key: { _id: 1 }, + unique: true + } ]; + + for (const row of result.rows) { + if (row.indexname === `${this._tableName}_pkey`) { + continue; + } + + const storedIndex = this._indexes.get(row.indexname); + if (storedIndex) { + indexes.push({ + name: storedIndex.mongoName || row.indexname, + key: storedIndex.keys, + unique: storedIndex.options.unique || false, + ...(storedIndex.options.sparse ? { sparse: true } : {}), + ...(storedIndex.options.type ? { type: storedIndex.options.type } : {}) + }); + } else { + indexes.push({ + name: row.indexname, + ...parseIndexDef(row.indexdef) + }); + } + } + + return indexes; + } + + async indexInformation() { + const indexes = await this.indexes(); + const info = {}; + for (const idx of indexes) { + info[idx.name] = Object.entries(idx.key).map(([ k, v ]) => [ k, v ]); + } + return info; + } + + // Legacy MongoDB method: insert (alias for insertOne or insertMany) + async insert(docs) { + if (Array.isArray(docs)) { + return this.insertMany(docs); + } + return this.insertOne(docs); + } + + initializeUnorderedBulkOp() { + const collection = this; + const operations = []; + + return { + find(query) { + return { + updateOne(update) { + operations.push({ + updateOne: { + filter: query, + update + } + }); + }, + update(update) { + operations.push({ + updateMany: { + filter: query, + update + } + }); + }, + upsert() { + return { + updateOne(update) { + operations.push({ + updateOne: { + filter: query, + update, + upsert: true + } + }); + }, + update(update) { + operations.push({ + updateMany: { + filter: query, + update, + upsert: true + } + }); + }, + replaceOne(doc) { + operations.push({ + replaceOne: { + filter: query, + replacement: doc, + upsert: true + } + }); + } + }; + }, + deleteOne() { + operations.push({ deleteOne: { filter: query } }); + }, + delete() { + operations.push({ deleteMany: { filter: query } }); + } + }; + }, + async execute() { + return collection.bulkWrite(operations); + } + }; + } + + async drop() { + const qualifiedName = this._qualifiedName(); + await this._pool.query(`DROP TABLE IF EXISTS ${qualifiedName}`); + this._initialized = false; + this._indexes.clear(); + } + + async rename(newName) { + const oldName = this._name; + const newCollName = validateTableName(newName); + const qualifiedName = this._qualifiedName(); + const escapedNewTableName = escapeIdentifier(newCollName); + await this._pool.query(`ALTER TABLE ${qualifiedName} RENAME TO "${escapedNewTableName}"`); + + // Update internal state + this._tableName = newCollName; + this._name = newName; + + // Update the database's collection cache + this._db._collections.delete(oldName); + this._db._collections.set(newName, this); + } +} + +// ============================================================================= +// Database Implementation +// ============================================================================= + +class PostgresDb { + constructor(client, name, schema) { + this._client = client; + this._pool = client._pool; + this._name = name; + this._schema = schema || null; + this._multiSchema = client._multiSchema || false; + this.databaseName = name; + this._collections = new Map(); + } + + collection(name) { + if (!this._collections.has(name)) { + this._collections.set(name, new PostgresCollection(this, name)); + } + return this._collections.get(name); + } + + async createCollection(name) { + const col = this.collection(name); + await col._ensureTable(); + return col; + } + + admin() { + const pool = this._pool; + const multiSchema = this._multiSchema; + const name = this._name; + const realDb = this._client._realDb; + return { + async listDatabases() { + if (multiSchema) { + // List all non-system schemas as virtual "databases", + // prefixed with the real PostgreSQL database name + const result = await pool.query(` + SELECT schema_name FROM information_schema.schemata + WHERE schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') + AND schema_name NOT LIKE 'pg_%' + `); + const databases = result.rows.map(row => ({ + name: realDb + '-' + row.schema_name + })); + return { databases }; + } + // Simple mode: just return this database + return { databases: [ { name } ] }; + } + }; + } + + async dropDatabase() { + if (!this._name) { + return; + } + if (this._multiSchema && this._schema) { + // Multi-schema mode: drop the schema + await this._pool.query( + `DROP SCHEMA IF EXISTS "${escapeIdentifier(this._schema)}" CASCADE` + ); + } else { + // Simple mode: drop all tables in public schema + const result = await this._pool.query( + 'SELECT tablename FROM pg_tables WHERE schemaname = \'public\'' + ); + for (const row of result.rows) { + await this._pool.query( + `DROP TABLE IF EXISTS "${escapeIdentifier(row.tablename)}" CASCADE` + ); + } + } + this._collections.clear(); + } + + async collections() { + const list = await this.listCollections().toArray(); + return list.map(entry => this.collection(entry.name)); + } + + listCollections() { + const self = this; + const schemaName = this._schema || 'public'; + return { + async toArray() { + const result = await self._pool.query(` + SELECT tablename as name + FROM pg_tables + WHERE schemaname = $1 + `, [ schemaName ]); + return result.rows.map(row => ({ + name: row.name + })); + } + }; + } +} + +// ============================================================================= +// Client Implementation +// ============================================================================= + +class PostgresClient { + constructor(pool, defaultDb, uri, options) { + this._pool = pool; + this._defaultDb = defaultDb; + this._uri = uri; + this._options = options; + this._multiSchema = options._multiSchema || false; + this._defaultSchema = options._defaultSchema || null; + this._realDb = options._realDb || null; + this._databases = new Map(); + } + + db(name) { + if (!this._multiSchema) { + // Simple mode: only the database from the connection URI is allowed. + if (name && name !== this._defaultDb) { + throw new Error( + `Cannot switch to database "${name}" in simple postgres:// mode.\n` + + 'All database names would share the same tables, causing data collisions.\n' + + 'Use a multipostgres:// URI for independent per-name data (via schemas).' + ); + } + if (!this._databases.has(this._defaultDb)) { + this._databases.set(this._defaultDb, new PostgresDb(this, this._defaultDb, null)); + } + return this._databases.get(this._defaultDb); + } + // Multi-schema mode: the virtual database name must start with + // the real PostgreSQL database name followed by a hyphen. + // The schema is derived from what follows that prefix. + const dbName = name || this._defaultDb; + const prefix = this._realDb + '-'; + if (!dbName.startsWith(prefix)) { + throw new Error( + `Invalid virtual database name "${dbName}": must start with "${prefix}".` + ); + } + const schema = dbName.substring(prefix.length); + if (!this._databases.has(dbName)) { + this._databases.set(dbName, new PostgresDb(this, dbName, schema)); + } + return this._databases.get(dbName); + } + + async close() { + if (!this._poolEnded) { + this._poolEnded = true; + await this._pool.end(); + } + } +} + +// ============================================================================= +// Module Export +// ============================================================================= + +module.exports = { + name: 'postgres', + // Native protocol schemes for this adapter + protocols: [ 'postgres', 'postgresql', 'multipostgres' ], + + /** + * Connect to PostgreSQL and return a client with MongoDB-compatible interface. + * + * Supports two modes: + * - postgres:// — Simple single-database mode, unprefixed tables in public schema + * - multipostgres:// — Multi-schema mode for multisite. + * URI: multipostgres://host/realdb-schemaname + * Last hyphen-separated component is the default schema, + * everything before is the real PostgreSQL database name. + * + * @param {string} uri - Connection URI + * @param {Object} [options={}] - Additional pg Pool options + * @returns {Promise} Client with db(), close() methods + */ + async connect(uri, options = {}) { + const url = new URL(uri); + let database; + let multiSchema = false; + let defaultSchema = null; + let realDb = null; + let connectionUri = uri; + + if (url.protocol === 'multipostgres:') { + // Multi-schema mode: multipostgres://host/realdb-schemaname + // The full path is the virtual "database name" (like MongoDB's database name). + // The real PostgreSQL database is everything before the last hyphen. + // The default schema is everything after the last hyphen. + multiSchema = true; + const path = url.pathname.slice(1); // e.g. 'shared-db-dashboard' + const lastHyphen = path.lastIndexOf('-'); + if (lastHyphen === -1) { + throw new Error( + 'multipostgres:// URI must contain at least one hyphen in the path: ' + + 'multipostgres://host/realdb-schemaname' + ); + } + realDb = path.substring(0, lastHyphen); + defaultSchema = path.substring(lastHyphen + 1); + // The virtual database name is the full path, matching MongoDB conventions + database = path; + + // Rewrite URI to postgres:// for the actual pg Pool connection + const connUrl = new URL(uri); + connUrl.protocol = 'postgres:'; + connUrl.pathname = '/' + realDb; + connectionUri = connUrl.toString(); + } else { + // Simple single-database mode + database = url.pathname.slice(1) || undefined; + } + + let pool = new Pool({ + connectionString: connectionUri, + ...options + }); + + // Test connection, creating the database if it doesn't exist + // (matching MongoDB's implicit database creation behavior) + try { + await pool.query('SELECT 1'); + } catch (e) { + // 3D000 = invalid_catalog_name (database does not exist) + const pgDatabase = new URL(connectionUri).pathname.slice(1); + if (e.code === '3D000' && pgDatabase) { + await pool.end(); + // Connect to the default 'postgres' database to create the target + const adminUrl = new URL(connectionUri); + adminUrl.pathname = '/postgres'; + const adminPool = new Pool({ + connectionString: adminUrl.toString(), + ...options + }); + try { + await adminPool.query(`CREATE DATABASE "${escapeIdentifier(pgDatabase)}"`); + } catch (createErr) { + // 42P04 = duplicate_database (another process just created it, that's fine) + if (createErr.code !== '42P04') { + throw createErr; + } + } finally { + await adminPool.end(); + } + // Reconnect to the now-existing database + pool = new Pool({ + connectionString: connectionUri, + ...options + }); + await pool.query('SELECT 1'); + } else { + throw e; + } + } + + return new PostgresClient(pool, database, uri, { + ...options, + _multiSchema: multiSchema, + _defaultSchema: defaultSchema, + _realDb: realDb + }); + }, + profileReport, + profileReset +}; diff --git a/packages/db-connect/adapters/sqlite.js b/packages/db-connect/adapters/sqlite.js new file mode 100644 index 0000000000..dbece06df6 --- /dev/null +++ b/packages/db-connect/adapters/sqlite.js @@ -0,0 +1,2398 @@ +// SQLite Adapter for MongoDB-compatible interface +// Stores documents as JSON text with _id as primary key +// Uses better-sqlite3 for synchronous, high-performance SQLite access + +const Database = require('better-sqlite3'); +const crypto = require('crypto'); +const path = require('path'); +const fs = require('fs'); +const { + serializeValue, + serializeDocument, + deserializeDocument, + getNestedField, + setNestedField, + deepEqual, + applyProjection, + applyUpdate, + extractAnchoredLiteralPrefix, + prefixUpperBound, + validateInteger +} = require('../lib/shared'); +const { AggregationCursor } = require('../lib/aggregation-cursor'); + +// ============================================================================= +// SECURITY: Input Validation and Escaping +// ============================================================================= + +const SAFE_IDENTIFIER_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + +function validateTableName(name) { + if (typeof name !== 'string' || name.length === 0 || name.length > 63) { + throw new Error('Invalid table name: must be a non-empty string up to 63 characters'); + } + const sanitized = name.replace(/-/g, '_'); + if (!SAFE_IDENTIFIER_PATTERN.test(sanitized)) { + throw new Error(`Invalid table name: "${name}" contains disallowed characters`); + } + return sanitized; +} + +// Sanitize a caller-supplied index name so it is safe to use as a SQLite +// identifier. Unlike validateTableName() (which rejects unsafe input as a +// security measure against malicious table names), index names frequently +// arrive from cross-backend JSONL dumps — MongoDB's default/auto-generated +// index names contain characters like "." that are illegal SQLite +// identifiers. Silently replacing those characters with "_" is safe: the +// name is an internal identifier, not user data, and the adapter is the +// one consulting the _indexes map by the sanitized form. +function sanitizeIndexName(name) { + if (typeof name !== 'string' || name.length === 0) { + return null; + } + const truncated = name.substring(0, 63); + const sanitized = truncated.replace(/[^a-zA-Z0-9_]/g, '_'); + // Identifiers must start with a letter or underscore. + if (/^[0-9]/.test(sanitized)) { + return '_' + sanitized.substring(0, 62); + } + return sanitized; +} + +function escapeIdentifier(name) { + return name.replace(/"/g, '""'); +} + +function escapeString(str) { + return str.replace(/'/g, '\'\''); +} + +// Convert a dot-path like "body.items.0.sublabel" to a SQLite JSON path +// like "$.body.items[0].sublabel". Numeric segments become array indices. +function toJsonPath(dotPath) { + return '$.' + dotPath.split('.').map(p => + /^\d+$/.test(p) ? `[${p}]` : escapeString(p) + ).join('.').replace(/\.\[/g, '['); +} + +// Generate a MongoDB-style ObjectId-like string +function generateId() { + return crypto.randomBytes(12).toString('hex'); +} + +// ============================================================================= +// SQLite Duplicate Key Error +// ============================================================================= + +function makeDuplicateKeyError(sqliteError, collection, doc) { + const message = 'Duplicate key error: already exists'; + const error = new Error(message); + error.code = 11000; + if (sqliteError.message) { + // For _id primary key: "UNIQUE constraint failed: tablename._id" + const colMatch = sqliteError.message.match(/UNIQUE constraint failed:\s*\S+\.(\S+)/); + if (colMatch && colMatch[1] === '_id') { + error.keyValue = { _id: doc ? doc._id : null }; + } + // For expression indexes: "UNIQUE constraint failed: index 'indexname'" + const idxMatch = sqliteError.message.match(/UNIQUE constraint failed:\s*index '([^']+)'/); + if (idxMatch && collection && collection._indexes) { + const indexMeta = collection._indexes.get(idxMatch[1]); + if (indexMeta && indexMeta.keys) { + error.keyValue = {}; + for (const field of Object.keys(indexMeta.keys)) { + error.keyValue[field] = doc ? getNestedField(doc, field) : null; + } + } + } + } + return error; +} + +// ============================================================================= +// Query Building for SQLite +// ============================================================================= + +// Build a json_extract path for a field: json_extract(data, '$.field.nested') +function buildJsonExtractPath(field) { + const parts = field.split('.'); + let path = '$'; + for (const p of parts) { + if (/^\d+$/.test(p)) { + // Array index — no dot before bracket notation + path += `[${p}]`; + } else { + path += `.${p.replace(/'/g, '\'\'')}`; + } + } + return `'${path}'`; +} + +// Build full json_extract expression +function buildJsonExtract(field, prefix = 'data') { + return `json_extract(${prefix}, ${buildJsonExtractPath(field)})`; +} + +/** + * Convert a MongoDB query object to a SQLite WHERE clause. + * + * MUTATES `params` by pushing values for parameterized query placeholders. + * The returned SQL string contains ? placeholders. + */ +// Build a SQL condition matching `textExpr` against `regex`. +// +// When the regex is anchored and begins with a literal prefix, emit +// `textExpr >= ? AND textExpr < upper(?) AND regexp(?, textExpr)`. The +// range predicate is btree-indexable on any expression index over +// `textExpr`; the residual regex call preserves correctness for any +// trailing pattern. MUTATES `params`. +function buildRegexMatchSql(textExpr, regex, params) { + const regexpFn = regex.ignoreCase ? 'regexp_i' : 'regexp'; + const { prefix } = extractAnchoredLiteralPrefix(regex); + const parts = []; + if (prefix) { + params.push(prefix); + parts.push(`${textExpr} >= ?`); + const upper = prefixUpperBound(prefix); + if (upper !== null) { + params.push(upper); + parts.push(`${textExpr} < ?`); + } + } + params.push(regex.source); + parts.push(`${regexpFn}(?, ${textExpr})`); + return parts.length > 1 ? `(${parts.join(' AND ')})` : parts[0]; +} + +function buildWhereClause(query, params, prefix = 'data', options = {}) { + const conditions = []; + + for (const [ key, value ] of Object.entries(query || {})) { + if (key === '$and') { + if (!Array.isArray(value)) { + throw new Error('$and must be an array'); + } + const andConditions = value.map(subQuery => { + const subClause = buildWhereClause(subQuery, params, prefix, options); + return `(${subClause})`; + }); + conditions.push(`(${andConditions.join(' AND ')})`); + } else if (key === '$or') { + if (!Array.isArray(value)) { + throw new Error('$or must be an array'); + } + const orConditions = value.map(subQuery => { + const subClause = buildWhereClause(subQuery, params, prefix, options); + return `(${subClause})`; + }); + conditions.push(`(${orConditions.join(' OR ')})`); + } else if (key === '$text') { + const searchTerm = value.$search; + if (typeof searchTerm !== 'string') { + throw new Error('$text.$search must be a string'); + } + const words = searchTerm.trim().split(/\s+/).filter(w => w.length > 0); + if (words.length === 0) { + conditions.push('0'); + } else if (options.ftsTable) { + // FTS5 search: match against the virtual table by rowid + // FTS5 query: OR the words together for MongoDB-compatible semantics + const ftsQuery = words.map(w => `"${w.replace(/"/g, '""')}"`).join(' OR '); + params.push(ftsQuery); + const rowidRef = options.mainTable ? `${options.mainTable}.rowid` : 'rowid'; + conditions.push( + `${rowidRef} IN (SELECT rowid FROM ${options.ftsTable} WHERE ${options.ftsTable} MATCH ?)` + ); + } else { + // Fallback LIKE-based search when no FTS5 table exists + const fieldNames = options.textFields || [ + 'highSearchText', 'lowSearchText', 'title', 'searchBoost' + ]; + const textFields = fieldNames.map(f => { + const jsonPath = '$.' + f.split('.').map(p => escapeString(p)).join('.'); + return `COALESCE(json_extract(${prefix}, '${jsonPath}'), '')`; + }); + const textExpr = textFields.join(' || \' \' || '); + const wordConditions = words.map(w => { + params.push(`%${w}%`); + return `(${textExpr}) LIKE ?`; + }); + conditions.push(`(${wordConditions.join(' OR ')})`); + } + } else if (key === '_id') { + if (value instanceof RegExp) { + conditions.push(buildRegexMatchSql('_id', value, params)); + } else if (typeof value === 'object' && value !== null && !Array.isArray(value)) { + conditions.push(buildOperatorClause('_id', value, params, true)); + } else { + params.push(value); + conditions.push('_id = ?'); + } + } else if (key.startsWith('$')) { + throw new Error(`Unsupported top-level operator: ${key}`); + } else if (typeof value === 'object' && value !== null && !Array.isArray(value) && !(value instanceof Date) && !(value instanceof RegExp)) { + const keys = Object.keys(value); + if (keys.some(k => k.startsWith('$'))) { + conditions.push(buildOperatorClause(key, value, params, false)); + } else { + // Nested object equality + params.push(JSON.stringify(value)); + conditions.push(`json_extract(${prefix}, '$.${escapeString(key)}') = ?`); + } + } else if (value instanceof RegExp) { + const jsonExtract = buildJsonExtract(key, prefix); + // Scalar match uses the indexable range + residual regex rewrite. + const scalarMatch = buildRegexMatchSql(jsonExtract, value, params); + // Array-element fallback (regex only — no per-element index to exploit). + const regexpFn = value.ignoreCase ? 'regexp_i' : 'regexp'; + params.push(value.source); + const arrayMatch = `(json_type(${prefix}, ${buildJsonExtractPath(key)}) = 'array' AND EXISTS(SELECT 1 FROM json_each(${jsonExtract}) WHERE ${regexpFn}(?, value)))`; + conditions.push(`(${scalarMatch} OR ${arrayMatch})`); + } else if (value === null || value === undefined) { + // MongoDB: { field: null } and { field: undefined } both match + // explicit null AND missing field + const jsonExtract = buildJsonExtract(key, prefix); + conditions.push(`(${jsonExtract} IS NULL OR json_type(${prefix}, ${buildJsonExtractPath(key)}) = 'null')`); + } else { + // Simple equality: handle both scalar equality AND array-contains-scalar + const jsonExtract = buildJsonExtract(key, prefix); + const serialized = serializeValue(value); + if (typeof serialized === 'boolean') { + // SQLite json_extract returns 1/0 for boolean + const boolVal = serialized ? 1 : 0; + params.push(boolVal); + params.push(JSON.stringify(serialized)); + conditions.push(`(${jsonExtract} = ? OR (json_type(${prefix}, ${buildJsonExtractPath(key)}) = 'array' AND EXISTS(SELECT 1 FROM json_each(${jsonExtract}) WHERE value = ?)))`); + } else if (typeof serialized === 'object' && serialized !== null) { + // Date or nested object + params.push(JSON.stringify(serialized)); + params.push(JSON.stringify(serialized)); + conditions.push(`(json(${jsonExtract}) = json(?) OR (json_type(${prefix}, ${buildJsonExtractPath(key)}) = 'array' AND EXISTS(SELECT 1 FROM json_each(${jsonExtract}) WHERE json(value) = json(?))))`); + } else { + params.push(serialized); + params.push(serialized); + conditions.push(`(${jsonExtract} = ? OR (json_type(${prefix}, ${buildJsonExtractPath(key)}) = 'array' AND EXISTS(SELECT 1 FROM json_each(${jsonExtract}) WHERE value = ?)))`); + } + } + } + + return conditions.length > 0 ? conditions.join(' AND ') : '1'; +} + +/** + * Build SQL conditions for MongoDB query operators. + */ +function buildOperatorClause(field, operators, params, isIdField = false) { + const conditions = []; + + const jsonExtract = isIdField ? '_id' : buildJsonExtract(field); + const jsonExtractPath = isIdField ? null : buildJsonExtractPath(field); + + for (const [ op, opValue ] of Object.entries(operators)) { + switch (op) { + case '$eq': + if (isIdField) { + if (opValue === null || opValue === undefined) { + conditions.push('_id IS NULL'); + } else { + params.push(opValue); + conditions.push('_id = ?'); + } + } else { + const serialized = serializeValue(opValue); + if (serialized === null || serialized === undefined) { + // SQL: = NULL is always false, must use IS NULL + conditions.push(`(${jsonExtract} IS NULL OR json_type(data, ${jsonExtractPath}) = 'null')`); + } else if (typeof serialized === 'object' && serialized !== null) { + params.push(JSON.stringify(serialized)); + conditions.push(`json(${jsonExtract}) = json(?)`); + } else if (typeof serialized === 'boolean') { + params.push(serialized ? 1 : 0); + conditions.push(`${jsonExtract} = ?`); + } else { + params.push(serialized); + conditions.push(`${jsonExtract} = ?`); + } + } + break; + + case '$ne': + if (isIdField) { + if (opValue === null || opValue === undefined) { + conditions.push('_id IS NOT NULL'); + } else { + params.push(opValue); + conditions.push('(_id IS NULL OR _id != ?)'); + } + } else { + const serialized = serializeValue(opValue); + if (serialized === null || serialized === undefined) { + // $ne: null means "field exists and is not null" + // SQL: != NULL is always false, must use IS NOT NULL + conditions.push(`(${jsonExtract} IS NOT NULL AND json_type(data, ${jsonExtractPath}) != 'null')`); + } else if (typeof serialized === 'object' && serialized !== null) { + params.push(JSON.stringify(serialized)); + conditions.push(`(${jsonExtract} IS NULL OR json(${jsonExtract}) != json(?))`); + } else if (typeof serialized === 'boolean') { + params.push(serialized ? 1 : 0); + conditions.push(`(${jsonExtract} IS NULL OR ${jsonExtract} != ?)`); + } else { + params.push(serialized); + conditions.push(`(${jsonExtract} IS NULL OR ${jsonExtract} != ?)`); + } + } + break; + + case '$gt': + if (isIdField) { + params.push(opValue); + conditions.push('_id > ?'); + } else if (opValue instanceof Date) { + params.push(opValue.toISOString()); + conditions.push(`json_extract(data, '$.${escapeString(field)}.$date') > ?`); + } else if (typeof opValue === 'string') { + params.push(opValue); + conditions.push(`${jsonExtract} > ?`); + } else { + params.push(opValue); + conditions.push(`CAST(${jsonExtract} AS NUMERIC) > ?`); + } + break; + + case '$gte': + if (isIdField) { + params.push(opValue); + conditions.push('_id >= ?'); + } else if (opValue instanceof Date) { + params.push(opValue.toISOString()); + conditions.push(`json_extract(data, '$.${escapeString(field)}.$date') >= ?`); + } else if (typeof opValue === 'string') { + params.push(opValue); + conditions.push(`${jsonExtract} >= ?`); + } else { + params.push(opValue); + conditions.push(`CAST(${jsonExtract} AS NUMERIC) >= ?`); + } + break; + + case '$lt': + if (isIdField) { + params.push(opValue); + conditions.push('_id < ?'); + } else if (opValue instanceof Date) { + params.push(opValue.toISOString()); + conditions.push(`json_extract(data, '$.${escapeString(field)}.$date') < ?`); + } else if (typeof opValue === 'string') { + params.push(opValue); + conditions.push(`${jsonExtract} < ?`); + } else { + params.push(opValue); + conditions.push(`CAST(${jsonExtract} AS NUMERIC) < ?`); + } + break; + + case '$lte': + if (isIdField) { + params.push(opValue); + conditions.push('_id <= ?'); + } else if (opValue instanceof Date) { + params.push(opValue.toISOString()); + conditions.push(`json_extract(data, '$.${escapeString(field)}.$date') <= ?`); + } else if (typeof opValue === 'string') { + params.push(opValue); + conditions.push(`${jsonExtract} <= ?`); + } else { + params.push(opValue); + conditions.push(`CAST(${jsonExtract} AS NUMERIC) <= ?`); + } + break; + + case '$in': + if (!Array.isArray(opValue)) { + throw new Error('$in requires an array'); + } + if (opValue.length === 0) { + conditions.push('0'); + } else if (isIdField) { + const placeholders = opValue.map(v => { + params.push(v); + return '?'; + }); + conditions.push(`_id IN (${placeholders.join(', ')})`); + } else { + const hasNull = opValue.includes(null); + const regexValues = opValue.filter(v => v instanceof RegExp); + const nonNullValues = opValue.filter(v => v !== null && !(v instanceof RegExp)); + const parts = []; + if (nonNullValues.length > 0) { + // For each value, check if the field equals it + // OR (if field is array) contains it + const valueParts = nonNullValues.map(v => { + const serialized = serializeValue(v); + if (typeof serialized === 'boolean') { + params.push(serialized ? 1 : 0); + return `${jsonExtract} = ?`; + } else if (typeof serialized === 'object' && serialized !== null) { + params.push(JSON.stringify(serialized)); + return `json(${jsonExtract}) = json(?)`; + } else { + params.push(serialized); + return `${jsonExtract} = ?`; + } + }); + // Also check if field is an array containing any of the values + const arrayParts = nonNullValues.map(v => { + const serialized = serializeValue(v); + if (typeof serialized === 'boolean') { + params.push(serialized ? 1 : 0); + return `EXISTS(SELECT 1 FROM json_each(${jsonExtract}) WHERE value = ?)`; + } else if (typeof serialized === 'object' && serialized !== null) { + params.push(JSON.stringify(serialized)); + return `EXISTS(SELECT 1 FROM json_each(${jsonExtract}) WHERE json(value) = json(?))`; + } else { + params.push(serialized); + return `EXISTS(SELECT 1 FROM json_each(${jsonExtract}) WHERE value = ?)`; + } + }); + parts.push(`(${valueParts.join(' OR ')} OR (json_type(data, ${jsonExtractPath}) = 'array' AND (${arrayParts.join(' OR ')})))`); + } + if (hasNull) { + parts.push(`(${jsonExtract} IS NULL OR json_type(data, ${jsonExtractPath}) = 'null')`); + } + // MongoDB supports RegExp values inside $in for pattern matching + for (const regex of regexValues) { + parts.push(buildRegexMatchSql(jsonExtract, regex, params)); + } + conditions.push(parts.length > 1 ? `(${parts.join(' OR ')})` : parts[0]); + } + break; + + case '$nin': + if (!Array.isArray(opValue)) { + throw new Error('$nin requires an array'); + } + if (opValue.length === 0) { + conditions.push('1'); + } else if (isIdField) { + const placeholders = opValue.map(v => { + params.push(v); + return '?'; + }); + conditions.push(`(_id IS NULL OR _id NOT IN (${placeholders.join(', ')}))`); + } else { + const hasNull = opValue.includes(null); + const nonNullValues = opValue.filter(v => v !== null); + const parts = []; + if (nonNullValues.length > 0) { + const valueParts = nonNullValues.map(v => { + const serialized = serializeValue(v); + if (typeof serialized === 'boolean') { + params.push(serialized ? 1 : 0); + return `${jsonExtract} != ?`; + } else if (typeof serialized === 'object' && serialized !== null) { + params.push(JSON.stringify(serialized)); + return `json(${jsonExtract}) != json(?)`; + } else { + params.push(serialized); + return `${jsonExtract} != ?`; + } + }); + parts.push(`(${jsonExtract} IS NULL OR (${valueParts.join(' AND ')}))`); + } + if (hasNull) { + parts.push(`(${jsonExtract} IS NOT NULL AND json_type(data, ${jsonExtractPath}) != 'null')`); + } else if (nonNullValues.length === 0) { + parts.push(`${jsonExtract} IS NULL`); + } + conditions.push(`(${parts.join(hasNull ? ' AND ' : ' OR ')})`); + } + break; + + case '$exists': + if (isIdField) { + conditions.push(opValue ? '_id IS NOT NULL' : '_id IS NULL'); + } else { + if (opValue) { + // $exists: true — field must be present in the JSON (even if null) + conditions.push(`json_type(data, ${jsonExtractPath}) IS NOT NULL`); + } else { + // $exists: false — field must be absent from the JSON + conditions.push(`json_type(data, ${jsonExtractPath}) IS NULL`); + } + } + break; + + case '$not': { + if (typeof opValue !== 'object' || opValue === null) { + throw new Error('$not requires an object'); + } + const negatedClause = buildOperatorClause( + field, opValue, params, isIdField + ); + conditions.push(`NOT (${negatedClause})`); + break; + } + + case '$regex': { + const pattern = opValue instanceof RegExp + ? opValue.source + : String(opValue); + const regexOptions = operators.$options || ''; + const caseInsensitive = regexOptions.includes('i'); + let regex; + try { + regex = new RegExp(pattern, caseInsensitive ? 'i' : ''); + } catch (e) { + // Fall back to direct emission if the pattern isn't valid + params.push(pattern); + const regexpFn = caseInsensitive ? 'regexp_i' : 'regexp'; + conditions.push( + isIdField ? `${regexpFn}(?, _id)` : `${regexpFn}(?, ${jsonExtract})` + ); + break; + } + conditions.push( + buildRegexMatchSql(isIdField ? '_id' : jsonExtract, regex, params) + ); + break; + } + + case '$options': + // Handled with $regex, skip + break; + + case '$all': + if (!Array.isArray(opValue)) { + throw new Error('$all requires an array'); + } + // Each value in the $all array must exist in the array field + for (const item of opValue) { + const serialized = serializeValue(item); + params.push(typeof serialized === 'object' && serialized !== null ? JSON.stringify(serialized) : serialized); + if (typeof serialized === 'boolean') { + conditions.push(`EXISTS(SELECT 1 FROM json_each(${jsonExtract}) WHERE value = ?)`); + } else if (typeof serialized === 'object' && serialized !== null) { + conditions.push(`EXISTS(SELECT 1 FROM json_each(${jsonExtract}) WHERE json(value) = json(?))`); + } else { + conditions.push(`EXISTS(SELECT 1 FROM json_each(${jsonExtract}) WHERE value = ?)`); + } + } + break; + + case '$size': + params.push(opValue); + conditions.push( + `json_type(data, ${jsonExtractPath}) = 'array' AND json_array_length(${jsonExtract}) = ?` + ); + break; + + default: + throw new Error(`Unsupported operator: ${op}`); + } + } + + return conditions.join(' AND '); +} + +// Build ORDER BY clause for SQLite +function queryHasText(query) { + if (!query || typeof query !== 'object' || !('$text' in query)) { + return false; + } + const search = query.$text && query.$text.$search; + if (typeof search !== 'string') { + return false; + } + return search.trim().split(/\s+/).filter(w => w.length > 0).length > 0; +} + +function buildOrderBy(sort, options = {}) { + const clauses = []; + + if (sort && Object.keys(sort).length > 0) { + for (const [ field, direction ] of Object.entries(sort)) { + if (direction && typeof direction === 'object' && direction.$meta === 'textScore') { + if (options.hasTextScore) { + clauses.push('_score DESC'); + } + continue; + } + if (direction && typeof direction === 'object' && direction.$meta) { + continue; + } + if (field === '_id') { + clauses.push(`_id ${direction === -1 ? 'DESC' : 'ASC'}`); + } else { + const jsonExtract = buildJsonExtract(field); + clauses.push(`${jsonExtract} ${direction === -1 ? 'DESC' : 'ASC'}`); + } + } + } + + // Use rowid as insertion-order tiebreaker (replaces postgres _order SERIAL). + // When joining with FTS5, qualify with the main table name to avoid ambiguity. + const rowidRef = options.mainTable ? `${options.mainTable}.rowid` : 'rowid'; + clauses.push(`${rowidRef} ASC`); + return `ORDER BY ${clauses.join(', ')}`; +} + + +// ============================================================================= +// Parse SQLite index definitions from sqlite_master +// ============================================================================= + +function parseIndexDef(sql) { + if (!sql) { + return { + key: {}, + unique: false + }; + } + const unique = /\bUNIQUE\b/i.test(sql); + const sparse = /\bWHERE\b/i.test(sql); + + const key = {}; + let type; + + // Extract expressions from CREATE INDEX ... ON tablename (expr1, expr2) + // Normalize whitespace to simplify matching across multi-line SQL + const normalizedSql = sql.replace(/\s+/g, ' ').trim(); + const onMatch = normalizedSql.match(/\bON\b\s+\S+\s*\((.+)\)(?:\s+WHERE\b.*)?$/i); + if (!onMatch) { + return { + key: {}, + unique, + ...(sparse ? { sparse: true } : {}) + }; + } + + const exprList = onMatch[1]; + const exprs = splitExpressions(exprList); + + for (const expr of exprs) { + // Strip outer parentheses that SQLite adds around expression indexes. + // e.g. "(json_extract(data, '$.slug')) ASC" → "json_extract(data, '$.slug') ASC" + let trimmed = stripOuterParens(expr.trim()); + + if (/^_id\b/.test(trimmed)) { + const direction = /\bDESC\b/i.test(trimmed) ? -1 : 1; + key._id = direction; + continue; + } + + // Numeric type: CAST(... AS NUMERIC) + if (/CAST\b.*\bAS\s+NUMERIC\b/i.test(trimmed)) { + type = 'number'; + const fieldName = jsonExtractToFieldName(trimmed); + if (fieldName) { + const direction = /\bDESC\b/i.test(trimmed) ? -1 : 1; + key[fieldName] = direction; + } + continue; + } + + // Date type: json_extract(data, '$.field.$date') + if (/\.\$date/i.test(trimmed)) { + type = 'date'; + const fieldName = jsonExtractToFieldName(trimmed); + if (fieldName) { + const direction = /\bDESC\b/i.test(trimmed) ? -1 : 1; + key[fieldName] = direction; + } + continue; + } + + // Text index (COALESCE pattern) + if (/\bCOALESCE\b/i.test(trimmed)) { + const fieldPattern = /json_extract\s*\(\s*data\s*,\s*'(\$\.[^']+)'\s*\)/gi; + let m; + while ((m = fieldPattern.exec(trimmed)) !== null) { + const path = m[1].replace(/^\$\./, ''); + key[path] = 'text'; + } + continue; + } + + // Default text type: json_extract(data, '$.field') + const fieldName = jsonExtractToFieldName(trimmed); + if (fieldName) { + const direction = /\bDESC\b/i.test(trimmed) ? -1 : 1; + key[fieldName] = direction; + } + } + + return { + key, + unique, + ...(sparse ? { sparse: true } : {}), + ...(type ? { type } : {}) + }; +} + +// Convert a json_extract expression back to a field name +function jsonExtractToFieldName(expr) { + const match = expr.match(/json_extract\s*\(\s*data\s*,\s*'(\$\.[^']+)'\s*\)/i); + if (!match) { + return null; + } + let path = match[1].replace(/^\$\./, ''); + // Remove .$date suffix for date indexes + path = path.replace(/\.\$date$/, ''); + return path || null; +} + +// Strip one layer of outer parentheses from an expression, preserving +// any trailing ASC/DESC. SQLite wraps expression-index columns in +// extra parens, e.g. "(json_extract(data, '$.slug')) ASC". +function stripOuterParens(str) { + if (!str.startsWith('(')) { + return str; + } + let depth = 0; + for (let i = 0; i < str.length; i++) { + if (str[i] === '(') { + depth++; + } else if (str[i] === ')') { + depth--; + if (depth === 0) { + const rest = str.slice(i + 1).trim(); + if (i > 0 && (/^(ASC|DESC)?$/i.test(rest))) { + return str.slice(1, i) + (rest ? ' ' + rest : ''); + } + return str; + } + } + } + return str; +} + +function splitExpressions(str) { + const results = []; + let depth = 0; + let current = ''; + for (const ch of str) { + if (ch === '(') { + depth++; + } else if (ch === ')') { + depth--; + } + if (ch === ',' && depth === 0) { + results.push(current); + current = ''; + } else { + current += ch; + } + } + if (current.trim()) { + results.push(current); + } + return results; +} + +// ============================================================================= +// Cursor Implementation +// ============================================================================= + +class SqliteCursor { + constructor(collection, query, options = {}) { + this._collection = collection; + this._query = query; + this._projection = options.projection || null; + this._sort = null; + this._limit = null; + this._skip = null; + this._iterator = null; + this._exhausted = false; + } + + project(projection) { + this._projection = projection; + return this; + } + + sort(sort) { + this._sort = sort; + return this; + } + + limit(n) { + const val = validateInteger(n, 'limit'); + this._limit = val === 0 ? null : val; + return this; + } + + skip(n) { + this._skip = validateInteger(n, 'skip'); + return this; + } + + clone() { + const cloned = new SqliteCursor(this._collection, this._query); + cloned._projection = this._projection; + cloned._sort = this._sort; + cloned._limit = this._limit; + cloned._skip = this._skip; + return cloned; + } + + // Build the SELECT SQL + params this cursor would execute. Shared by + // toArray() and exposed via explain() for EXPLAIN-based tests and + // debugging. SQL uses SQLite's native `?` placeholders; `params` is a + // positional array in order. + _buildFindSql() { + const params = []; + const queryOptions = this._collection._queryOptions(); + const tableName = this._collection._quotedTableName(); + const whereClause = buildWhereClause(this._query, params, 'data', queryOptions); + const hasText = queryHasText(this._query) && queryOptions.ftsTable; + const orderBy = buildOrderBy(this._sort, { + hasTextScore: hasText, + mainTable: hasText ? tableName : null + }); + + let selectCols = `${tableName}._id, ${tableName}.data`; + let sql; + if (hasText) { + // Build WHERE from query without $text (the FTS JOIN handles text matching) + const nonTextQuery = { ...this._query }; + delete nonTextQuery.$text; + const nonTextParams = []; + const nonTextWhere = buildWhereClause(nonTextQuery, nonTextParams, 'data', queryOptions); + + const words = this._query.$text.$search.trim().split(/\s+/).filter(w => w.length > 0); + const ftsQuery = words.map(w => `"${w.replace(/"/g, '""')}"`).join(' OR '); + + // bm25() returns negative values (lower = better), so negate for + // higher = better, matching MongoDB/PostgreSQL conventions + selectCols += `, -bm25(${queryOptions.ftsTable}) AS _score`; + + sql = `SELECT ${selectCols} FROM ${tableName} JOIN ${queryOptions.ftsTable} ON ${tableName}.rowid = ${queryOptions.ftsTable}.rowid WHERE ${queryOptions.ftsTable} MATCH ? AND ${nonTextWhere} ${orderBy}`; + // Replace params with: ftsQuery first, then nonTextParams + params.length = 0; + params.push(ftsQuery, ...nonTextParams); + } else { + sql = `SELECT ${selectCols} FROM ${tableName} WHERE ${whereClause} ${orderBy}`; + } + + if (this._limit != null) { + sql += ` LIMIT ${this._limit}`; + } else if (this._skip != null) { + sql += ' LIMIT -1'; + } + if (this._skip != null) { + sql += ` OFFSET ${this._skip}`; + } + return { + sql, + params + }; + } + + // Returns the SQL and parameter values the adapter would execute for + // this cursor's current query/sort/limit/skip/projection. Useful for + // EXPLAIN-based tests and for debugging query planner behavior. The + // returned SQL uses SQLite's native `?` placeholder style. + async explain() { + this._collection._ensureTable(); + return this._buildFindSql(); + } + + async toArray() { + this._collection._ensureTable(); + + const { sql, params } = this._buildFindSql(); + const rows = this._collection._db._sqlite.prepare(sql).all(...params); + return rows.map(row => { + const doc = deserializeDocument(row.data, row._id); + const meta = row._score != null ? { textScore: row._score } : {}; + return this._projection ? applyProjection(doc, this._projection, meta) : doc; + }); + } + + next(callback) { + const promise = this._next(); + if (callback) { + promise.then(doc => callback(null, doc), err => callback(err)); + return; + } + return promise; + } + + async _next() { + if (this._exhausted) { + return null; + } + // Buffer all results on first call to avoid holding the database busy. + // better-sqlite3 is synchronous and an active .iterate() cursor blocks + // all other queries on the same connection. + if (!this._buffer) { + this._collection._ensureTable(); + + const params = []; + const queryOptions = this._collection._queryOptions(); + const tableName = this._collection._quotedTableName(); + const whereClause = buildWhereClause(this._query, params, 'data', queryOptions); + const hasText = queryHasText(this._query) && queryOptions.ftsTable; + const orderBy = buildOrderBy(this._sort, { + hasTextScore: hasText, + mainTable: hasText ? tableName : null + }); + + let selectCols = `${tableName}._id, ${tableName}.data`; + let sql; + if (hasText) { + const nonTextQuery = { ...this._query }; + delete nonTextQuery.$text; + const nonTextParams = []; + const nonTextWhere = buildWhereClause(nonTextQuery, nonTextParams, 'data', queryOptions); + + const words = this._query.$text.$search.trim().split(/\s+/).filter(w => w.length > 0); + const ftsQuery = words.map(w => `"${w.replace(/"/g, '""')}"`).join(' OR '); + + selectCols += `, -bm25(${queryOptions.ftsTable}) AS _score`; + sql = `SELECT ${selectCols} FROM ${tableName} JOIN ${queryOptions.ftsTable} ON ${tableName}.rowid = ${queryOptions.ftsTable}.rowid WHERE ${queryOptions.ftsTable} MATCH ? AND ${nonTextWhere} ${orderBy}`; + params.length = 0; + params.push(ftsQuery, ...nonTextParams); + } else { + sql = `SELECT ${selectCols} FROM ${tableName} WHERE ${whereClause} ${orderBy}`; + } + + if (this._limit != null) { + sql += ` LIMIT ${this._limit}`; + } else if (this._skip != null) { + sql += ' LIMIT -1'; + } + if (this._skip != null) { + sql += ` OFFSET ${this._skip}`; + } + + this._buffer = this._collection._db._sqlite.prepare(sql).all(...params); + this._bufferIndex = 0; + } + + if (this._bufferIndex >= this._buffer.length) { + this._exhausted = true; + this._buffer = null; + return null; + } + + const row = this._buffer[this._bufferIndex++]; + const doc = deserializeDocument(row.data, row._id); + const meta = row._score != null ? { textScore: row._score } : {}; + return this._projection ? applyProjection(doc, this._projection, meta) : doc; + } + + async hasNext() { + if (this._exhausted) { + return false; + } + if (!this._buffer) { + // Force buffering by calling _next logic without consuming + await this._next(); + if (this._exhausted) { + return false; + } + // Put the item back + this._bufferIndex--; + return true; + } + return this._bufferIndex < this._buffer.length; + } + + async close() { + this._buffer = null; + this._exhausted = true; + } + + addCursorFlag() { + return this; + } + + [Symbol.asyncIterator]() { + return { + cursor: this, + async next() { + const doc = await this.cursor._next(); + if (doc === null) { + return { + done: true, + value: undefined + }; + } + return { + done: false, + value: doc + }; + } + }; + } + + async count() { + this._collection._ensureTable(); + + const params = []; + const tableName = this._collection._quotedTableName(); + const whereClause = buildWhereClause(this._query, params, 'data', this._collection._queryOptions()); + const sql = `SELECT COUNT(*) as count FROM ${tableName} WHERE ${whereClause}`; + const row = this._collection._db._sqlite.prepare(sql).get(...params); + return row.count; + } +} + +// ============================================================================= +// Collection Implementation +// ============================================================================= + +class SqliteCollection { + constructor(db, name) { + this._db = db; + this._tableName = validateTableName(name); + this._name = name; + this._indexes = new Map(); + this._textFields = null; + this._initialized = false; + } + + _quotedTableName() { + return `"${escapeIdentifier(this._tableName)}"`; + } + + _ftsTableName() { + return `"${escapeIdentifier(this._tableName + '_fts')}"`; + } + + _queryOptions() { + if (!this._textFields) { + return {}; + } + return { + textFields: this._textFields, + ftsTable: this._ftsTableName(), + mainTable: this._quotedTableName() + }; + } + + // Extract text field values from a document for FTS5 indexing. + // Returns an array of string values in _textFields order, or null + // if no FTS5 table exists. + _extractFtsValues(doc) { + if (!this._textFields) { + return null; + } + return this._textFields.map(f => { + const val = getNestedField(doc, f); + return val != null ? String(val) : ''; + }); + } + + // Sync FTS5 table after inserting a document + _syncFtsInsert(id, doc) { + const values = this._extractFtsValues(doc); + if (!values) { + return; + } + const fts = this._ftsTableName(); + const placeholders = values.map(() => '?').join(', '); + this._db._sqlite.prepare( + `INSERT INTO ${fts} (rowid, ${this._textFields.map(f => `"${escapeIdentifier(f)}"`).join(', ')}) VALUES ((SELECT rowid FROM ${this._quotedTableName()} WHERE _id = ?), ${placeholders})` + ).run(id, ...values); + } + + // Delete a document's FTS5 entry by its _id. + // With contentless_delete=1, we can delete by rowid without old values. + _syncFtsDelete(id) { + if (!this._textFields) { + return; + } + const tableName = this._quotedTableName(); + const fts = this._ftsTableName(); + const row = this._db._sqlite.prepare( + `SELECT rowid FROM ${tableName} WHERE _id = ?` + ).get(id); + if (!row) { + return; + } + this._db._sqlite.prepare( + `DELETE FROM ${fts} WHERE rowid = ?` + ).run(row.rowid); + } + + get collectionName() { + return this._name; + } + + get name() { + return this._name; + } + + _ensureTable() { + if (this._initialized) { + return; + } + + const tableName = this._quotedTableName(); + this._db._sqlite.exec(` + CREATE TABLE IF NOT EXISTS ${tableName} ( + _id TEXT PRIMARY KEY, + data TEXT NOT NULL + ) + `); + this._initialized = true; + } + + async insertOne(doc) { + this._ensureTable(); + + const id = doc._id != null ? String(doc._id) : generateId(); + const docWithoutId = { ...doc }; + delete docWithoutId._id; + + const tableName = this._quotedTableName(); + try { + this._db._sqlite.prepare( + `INSERT INTO ${tableName} (_id, data) VALUES (?, ?)` + ).run(id, serializeDocument(docWithoutId)); + this._syncFtsInsert(id, doc); + return { + acknowledged: true, + insertedId: id, + insertedCount: 1, + ops: [ { + ...doc, + _id: id + } ], + result: { ok: 1 } + }; + } catch (e) { + if (e.code === 'SQLITE_CONSTRAINT_PRIMARYKEY' || e.code === 'SQLITE_CONSTRAINT_UNIQUE' || (e.message && e.message.includes('UNIQUE constraint failed'))) { + throw makeDuplicateKeyError(e, this, { + ...doc, + _id: id + }); + } + throw e; + } + } + + async insertMany(docs) { + this._ensureTable(); + + const insertedIds = {}; + let insertedCount = 0; + + for (let i = 0; i < docs.length; i++) { + const result = await this.insertOne(docs[i]); + insertedIds[i] = result.insertedId; + insertedCount++; + } + + return { + acknowledged: true, + insertedCount, + insertedIds, + result: { ok: 1 } + }; + } + + async findOne(query, options = {}) { + this._ensureTable(); + + const params = []; + const tableName = this._quotedTableName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + const sql = `SELECT _id, data FROM ${tableName} WHERE ${whereClause} LIMIT 1`; + + const row = this._db._sqlite.prepare(sql).get(...params); + if (!row) { + return null; + } + + const doc = deserializeDocument(row.data, row._id); + return options.projection ? applyProjection(doc, options.projection) : doc; + } + + find(query, options) { + return new SqliteCursor(this, query, options); + } + + async updateOne(query, update, options = {}) { + this._ensureTable(); + + if (typeof options === 'function') { + options = {}; + } + + // Single-statement fast path: when the update uses only simple + // operators without upsert, execute a single UPDATE statement + // instead of the read-modify-write cycle. $push, $pull and + // $addToSet are included only when all their values are scalars + // (strings, numbers, booleans). Skip if any touched field is + // text-indexed (FTS sync requires the full read-modify-write path). + if (!options.upsert) { + const atomicOps = [ + '$inc', '$set', '$unset', '$currentDate', + '$push', '$pull', '$addToSet' + ]; + const ops = Object.keys(update); + const isAtomicCompatible = ops.length > 0 && + ops.every(op => atomicOps.includes(op)); + if (isAtomicCompatible) { + // $push/$pull/$addToSet only qualify when all values are scalars + const allScalar = [ '$push', '$pull', '$addToSet' ].every(op => { + if (!update[op]) { + return true; + } + return Object.values(update[op]).every(v => + typeof v === 'string' || + typeof v === 'number' || + typeof v === 'boolean' + ); + }); + if (allScalar) { + const textFields = this._textFields || []; + const unsetKeys = Array.isArray(update.$unset) + ? update.$unset + : Object.keys(update.$unset || {}); + const allKeys = [ + ...Object.keys(update.$set || {}), + ...Object.keys(update.$inc || {}), + ...unsetKeys, + ...Object.keys(update.$currentDate || {}), + ...Object.keys(update.$push || {}), + ...Object.keys(update.$pull || {}), + ...Object.keys(update.$addToSet || {}) + ]; + const touchesTextFields = + allKeys.some(f => textFields.includes(f)); + if (!touchesTextFields) { + return this._atomicUpdateOne(query, update); + } + } + } + } + + const params = []; + const tableName = this._quotedTableName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + const selectSql = `SELECT _id, data FROM ${tableName} WHERE ${whereClause} LIMIT 1`; + const selectResult = this._db._sqlite.prepare(selectSql).get(...params); + + if (!selectResult) { + if (options.upsert) { + let newDoc = {}; + if (query._id) { + newDoc._id = query._id; + } + newDoc = applyUpdate(newDoc, update); + const insertResult = await this.insertOne(newDoc); + return { + acknowledged: true, + matchedCount: 0, + modifiedCount: 0, + upsertedId: insertResult.insertedId, + upsertedCount: 1, + result: { + nModified: 0, + n: 1 + } + }; + } + return { + acknowledged: true, + matchedCount: 0, + modifiedCount: 0, + result: { + nModified: 0, + n: 0 + } + }; + } + + const existing = deserializeDocument(selectResult.data, selectResult._id); + const updated = applyUpdate(existing, update); + const { _id, ...dataWithoutId } = updated; + + try { + this._syncFtsDelete(selectResult._id); + this._db._sqlite.prepare( + `UPDATE ${tableName} SET data = ? WHERE _id = ?` + ).run(serializeDocument(dataWithoutId), selectResult._id); + this._syncFtsInsert(selectResult._id, updated); + } catch (e) { + if (e.code === 'SQLITE_CONSTRAINT_PRIMARYKEY' || e.code === 'SQLITE_CONSTRAINT_UNIQUE' || (e.message && e.message.includes('UNIQUE constraint failed'))) { + throw makeDuplicateKeyError(e, this, updated); + } + throw e; + } + + return { + acknowledged: true, + matchedCount: 1, + modifiedCount: 1, + result: { + nModified: 1, + n: 1 + } + }; + } + + _atomicUpdateOne(query, update) { + const tableName = this._quotedTableName(); + + // Build SET expression and its params first (they appear before WHERE in SQL) + const setParams = []; + let dataExpr = 'data'; + + if (update.$set) { + for (const [ field, value ] of Object.entries(update.$set)) { + const jsonPath = toJsonPath(field); + const serialized = serializeValue(value); + setParams.push(JSON.stringify(serialized)); + dataExpr = `json_set(${dataExpr}, '${jsonPath}', json(?))`; + } + } + + if (update.$inc) { + for (const [ field, value ] of Object.entries(update.$inc)) { + const jsonPath = toJsonPath(field); + setParams.push(value); + dataExpr = `json_set(${dataExpr}, '${jsonPath}', COALESCE(json_extract(data, '${jsonPath}'), 0) + ?)`; + } + } + + if (update.$unset) { + const fields = Array.isArray(update.$unset) + ? update.$unset + : Object.keys(update.$unset); + for (const field of fields) { + const jsonPath = toJsonPath(field); + dataExpr = `json_remove(${dataExpr}, '${jsonPath}')`; + } + } + + if (update.$currentDate) { + for (const [ field, value ] of Object.entries(update.$currentDate)) { + if (value === true || (value && value.$type === 'date')) { + const jsonPath = toJsonPath(field); + const dateVal = JSON.stringify(serializeValue(new Date())); + setParams.push(dateVal); + dataExpr = `json_set(${dataExpr}, '${jsonPath}', json(?))`; + } + } + } + + // $push: append scalar value to array + if (update.$push) { + for (const [ field, value ] of Object.entries(update.$push)) { + const jsonPath = toJsonPath(field); + setParams.push(value); + const coalesced = `COALESCE(json_extract(data, '${jsonPath}'), json('[]'))`; + dataExpr = `json_set(${dataExpr}, '${jsonPath}', json_insert(${coalesced}, '$[#]', ?))`; + } + } + + // $pull: remove scalar value from array + if (update.$pull) { + for (const [ field, value ] of Object.entries(update.$pull)) { + const jsonPath = toJsonPath(field); + setParams.push(value); + dataExpr = `json_set(${dataExpr}, '${jsonPath}', ` + + '(SELECT json_group_array(je.value) ' + + `FROM json_each(COALESCE(json_extract(data, '${jsonPath}'), '[]')) AS je ` + + 'WHERE je.value != ?))'; + } + } + + // $addToSet: add scalar value to array if not already present + if (update.$addToSet) { + for (const [ field, value ] of Object.entries(update.$addToSet)) { + const jsonPath = toJsonPath(field); + setParams.push(value, value); + const coalesced = `COALESCE(json_extract(data, '${jsonPath}'), json('[]'))`; + dataExpr = `json_set(${dataExpr}, '${jsonPath}', ` + + `CASE WHEN EXISTS(SELECT 1 FROM json_each(${coalesced}) WHERE value = ?) ` + + `THEN ${coalesced} ` + + `ELSE json_insert(${coalesced}, '$[#]', ?) END)`; + } + } + + // Build WHERE clause params second + const whereParams = []; + const whereClause = buildWhereClause(query, whereParams, 'data', this._queryOptions()); + + // Positional params: SET params first, then WHERE params + const params = [ ...setParams, ...whereParams ]; + const sql = `UPDATE ${tableName} SET data = ${dataExpr} WHERE ${whereClause}`; + try { + const result = this._db._sqlite.prepare(sql).run(...params); + const matched = result.changes > 0 ? 1 : 0; + return { + acknowledged: true, + matchedCount: matched, + modifiedCount: matched, + result: { + nModified: matched, + n: matched + } + }; + } catch (e) { + if (e.code === 'SQLITE_CONSTRAINT_PRIMARYKEY' || e.code === 'SQLITE_CONSTRAINT_UNIQUE' || (e.message && e.message.includes('UNIQUE constraint failed'))) { + // Build a pseudo-doc from $set values so makeDuplicateKeyError + // can report the conflicting key values + const pseudoDoc = {}; + if (update.$set) { + for (const [ field, value ] of Object.entries(update.$set)) { + setNestedField(pseudoDoc, field, value); + } + } + throw makeDuplicateKeyError(e, this, pseudoDoc); + } + throw e; + } + } + + async updateMany(query, update, options = {}) { + this._ensureTable(); + + const params = []; + const tableName = this._quotedTableName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + const selectSql = `SELECT _id, data FROM ${tableName} WHERE ${whereClause}`; + const rows = this._db._sqlite.prepare(selectSql).all(...params); + + if (rows.length === 0) { + return { + acknowledged: true, + matchedCount: 0, + modifiedCount: 0, + result: { + nModified: 0, + n: 0 + } + }; + } + + let modifiedCount = 0; + const updateStmt = this._db._sqlite.prepare( + `UPDATE ${tableName} SET data = ? WHERE _id = ?` + ); + + for (const row of rows) { + const existing = deserializeDocument(row.data, row._id); + const updated = applyUpdate(existing, update); + const { _id, ...dataWithoutId } = updated; + this._syncFtsDelete(row._id); + updateStmt.run(serializeDocument(dataWithoutId), row._id); + this._syncFtsInsert(row._id, updated); + modifiedCount++; + } + + return { + acknowledged: true, + matchedCount: rows.length, + modifiedCount, + result: { + nModified: modifiedCount, + n: rows.length + } + }; + } + + async replaceOne(query, replacement, options = {}) { + this._ensureTable(); + + const params = []; + const tableName = this._quotedTableName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + const selectSql = `SELECT _id FROM ${tableName} WHERE ${whereClause} LIMIT 1`; + const selectResult = this._db._sqlite.prepare(selectSql).get(...params); + + if (!selectResult) { + if (options.upsert) { + const result = await this.insertOne(replacement); + return { + acknowledged: true, + matchedCount: 0, + modifiedCount: 0, + upsertedId: result.insertedId, + upsertedCount: 1 + }; + } + return { + acknowledged: true, + matchedCount: 0, + modifiedCount: 0 + }; + } + + const { _id, ...dataWithoutId } = replacement; + try { + this._syncFtsDelete(selectResult._id); + this._db._sqlite.prepare( + `UPDATE ${tableName} SET data = ? WHERE _id = ?` + ).run(serializeDocument(dataWithoutId), selectResult._id); + this._syncFtsInsert(selectResult._id, replacement); + } catch (e) { + if (e.code === 'SQLITE_CONSTRAINT_PRIMARYKEY' || e.code === 'SQLITE_CONSTRAINT_UNIQUE' || (e.message && e.message.includes('UNIQUE constraint failed'))) { + throw makeDuplicateKeyError(e, this, replacement); + } + throw e; + } + + return { + acknowledged: true, + matchedCount: 1, + modifiedCount: 1 + }; + } + + async deleteOne(query) { + this._ensureTable(); + + const params = []; + const tableName = this._quotedTableName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + if (this._textFields) { + const fts = this._ftsTableName(); + const row = this._db._sqlite.prepare( + `SELECT rowid FROM ${tableName} WHERE ${whereClause} LIMIT 1` + ).get(...params); + if (!row) { + return { + acknowledged: true, + deletedCount: 0, + result: { ok: 1 } + }; + } + this._db._sqlite.prepare(`DELETE FROM ${fts} WHERE rowid = ?`).run(row.rowid); + this._db._sqlite.prepare(`DELETE FROM ${tableName} WHERE rowid = ?`).run(row.rowid); + return { + acknowledged: true, + deletedCount: 1, + result: { ok: 1 } + }; + } + + // Even without _textFields set, clean up FTS entries if the FTS table exists + const fts = this._ftsTableName(); + const ftsExists = this._db._sqlite.prepare( + 'SELECT 1 FROM sqlite_master WHERE type=\'table\' AND name=?' + ).get(this._tableName + '_fts'); + if (ftsExists) { + const row = this._db._sqlite.prepare( + `SELECT rowid FROM ${tableName} WHERE ${whereClause} LIMIT 1` + ).get(...params); + if (!row) { + return { + acknowledged: true, + deletedCount: 0, + result: { ok: 1 } + }; + } + this._db._sqlite.prepare(`DELETE FROM ${fts} WHERE rowid = ?`).run(row.rowid); + this._db._sqlite.prepare(`DELETE FROM ${tableName} WHERE rowid = ?`).run(row.rowid); + return { + acknowledged: true, + deletedCount: 1, + result: { ok: 1 } + }; + } + + const result = this._db._sqlite.prepare( + `DELETE FROM ${tableName} WHERE _id IN ( + SELECT _id FROM ${tableName} WHERE ${whereClause} LIMIT 1 + )` + ).run(...params); + + return { + acknowledged: true, + deletedCount: result.changes, + result: { ok: 1 } + }; + } + + async deleteMany(query) { + this._ensureTable(); + + const params = []; + const tableName = this._quotedTableName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + if (this._textFields) { + const fts = this._ftsTableName(); + const rows = this._db._sqlite.prepare( + `SELECT rowid FROM ${tableName} WHERE ${whereClause}` + ).all(...params); + if (rows.length > 0) { + const deleteFts = this._db._sqlite.prepare(`DELETE FROM ${fts} WHERE rowid = ?`); + const deleteMain = this._db._sqlite.prepare(`DELETE FROM ${tableName} WHERE rowid = ?`); + for (const row of rows) { + deleteFts.run(row.rowid); + deleteMain.run(row.rowid); + } + } + return { + acknowledged: true, + deletedCount: rows.length, + result: { ok: 1 } + }; + } + + // Even without _textFields set (e.g. a separate connection that never + // called createIndex), clean up FTS entries if the FTS table exists. + // This prevents stale FTS data when external tools delete documents. + const fts = this._ftsTableName(); + const ftsExists = this._db._sqlite.prepare( + 'SELECT 1 FROM sqlite_master WHERE type=\'table\' AND name=?' + ).get(this._tableName + '_fts'); + if (ftsExists) { + const rows = this._db._sqlite.prepare( + `SELECT rowid FROM ${tableName} WHERE ${whereClause}` + ).all(...params); + if (rows.length > 0) { + const deleteFts = this._db._sqlite.prepare(`DELETE FROM ${fts} WHERE rowid = ?`); + const deleteMain = this._db._sqlite.prepare(`DELETE FROM ${tableName} WHERE rowid = ?`); + for (const row of rows) { + deleteFts.run(row.rowid); + deleteMain.run(row.rowid); + } + } + return { + acknowledged: true, + deletedCount: rows.length, + result: { ok: 1 } + }; + } + + const result = this._db._sqlite.prepare( + `DELETE FROM ${tableName} WHERE ${whereClause}` + ).run(...params); + + return { + acknowledged: true, + deletedCount: result.changes, + result: { ok: 1 } + }; + } + + async remove(query) { + return this.deleteMany(query); + } + + async removeOne(query) { + return this.deleteOne(query); + } + + async removeMany(query) { + return this.deleteMany(query); + } + + async countDocuments(query = {}) { + this._ensureTable(); + + const params = []; + const tableName = this._quotedTableName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + const sql = `SELECT COUNT(*) as count FROM ${tableName} WHERE ${whereClause}`; + + const row = this._db._sqlite.prepare(sql).get(...params); + return row.count; + } + + async distinct(field, query = {}) { + this._ensureTable(); + + const params = []; + const tableName = this._quotedTableName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + if (field === '_id') { + const sql = `SELECT DISTINCT _id as value FROM ${tableName} WHERE ${whereClause}`; + const rows = this._db._sqlite.prepare(sql).all(...params); + return rows.map(row => row.value).filter(v => v !== null); + } + + const jsonExtract = buildJsonExtract(field); + const jsonExtractPath = buildJsonExtractPath(field); + // Flatten arrays like MongoDB's distinct(), preserving type info for booleans. + // For arrays: json_each provides type; for scalars: json_type on the field itself. + // Use UNION to combine both cases. + const sql = ` + SELECT DISTINCT je.value as value, je.type as type FROM ${tableName}, json_each(${jsonExtract}) AS je + WHERE ${whereClause} AND json_type(data, ${jsonExtractPath}) = 'array' + UNION + SELECT DISTINCT ${jsonExtract} as value, json_type(data, ${jsonExtractPath}) as type FROM ${tableName} + WHERE ${whereClause} AND ${jsonExtract} IS NOT NULL AND json_type(data, ${jsonExtractPath}) != 'array' + `; + + const rows = this._db._sqlite.prepare(sql).all(...params, ...params); + + return rows + .map(row => { + const v = row.value; + if (v === null || v === undefined) { + return null; + } + // Convert SQLite boolean representations back to JS booleans + // json_type returns 'true'/'false' for boolean values + if (row.type === 'true') { + return true; + } + if (row.type === 'false') { + return false; + } + // Parse JSON objects and arrays back to JS values + if (row.type === 'object' || row.type === 'array') { + return JSON.parse(v); + } + return v; + }) + .filter(v => v !== null); + } + + aggregate(pipeline) { + return new AggregationCursor(this, pipeline); + } + + async bulkWrite(operations) { + this._ensureTable(); + + let insertedCount = 0; + let modifiedCount = 0; + let deletedCount = 0; + let upsertedCount = 0; + const insertedIds = {}; + const upsertedIds = {}; + + for (let i = 0; i < operations.length; i++) { + const op = operations[i]; + + if (op.insertOne) { + const result = await this.insertOne(op.insertOne.document); + insertedIds[i] = result.insertedId; + insertedCount++; + } else if (op.updateOne) { + const result = await this.updateOne( + op.updateOne.filter, + op.updateOne.update, + { upsert: op.updateOne.upsert } + ); + modifiedCount += result.modifiedCount; + if (result.upsertedId) { + upsertedIds[i] = result.upsertedId; + upsertedCount++; + } + } else if (op.updateMany) { + const result = await this.updateMany( + op.updateMany.filter, + op.updateMany.update, + { upsert: op.updateMany.upsert } + ); + modifiedCount += result.modifiedCount; + } else if (op.deleteOne) { + const result = await this.deleteOne(op.deleteOne.filter); + deletedCount += result.deletedCount; + } else if (op.deleteMany) { + const result = await this.deleteMany(op.deleteMany.filter); + deletedCount += result.deletedCount; + } else if (op.replaceOne) { + const result = await this.replaceOne( + op.replaceOne.filter, + op.replaceOne.replacement, + { upsert: op.replaceOne.upsert } + ); + modifiedCount += result.modifiedCount; + if (result.upsertedId) { + upsertedIds[i] = result.upsertedId; + upsertedCount++; + } + } + } + + return { + acknowledged: true, + insertedCount, + modifiedCount, + deletedCount, + upsertedCount, + insertedIds, + upsertedIds + }; + } + + async findOneAndUpdate(query, update, options = {}) { + this._ensureTable(); + + const params = []; + const tableName = this._quotedTableName(); + const whereClause = buildWhereClause(query, params, 'data', this._queryOptions()); + + const selectSql = `SELECT _id, data FROM ${tableName} WHERE ${whereClause} LIMIT 1`; + const selectResult = this._db._sqlite.prepare(selectSql).get(...params); + + if (!selectResult) { + if (options.upsert) { + let newDoc = {}; + if (query._id) { + newDoc._id = query._id; + } + newDoc = applyUpdate(newDoc, update); + await this.insertOne(newDoc); + return options.returnDocument === 'after' ? newDoc : null; + } + return null; + } + + const existing = deserializeDocument(selectResult.data, selectResult._id); + const updated = applyUpdate(existing, update); + const { _id, ...dataWithoutId } = updated; + + this._syncFtsDelete(selectResult._id); + this._db._sqlite.prepare( + `UPDATE ${tableName} SET data = ? WHERE _id = ?` + ).run(serializeDocument(dataWithoutId), selectResult._id); + this._syncFtsInsert(selectResult._id, updated); + + return options.returnDocument === 'after' ? updated : existing; + } + + async createIndex(keys, options = {}) { + this._ensureTable(); + + const keyEntries = Object.entries(keys); + const indexType = options.type; + + const buildIndexPath = (field, type) => { + if (type === 'date') { + return `json_extract(data, '$.${escapeString(field)}.$date')`; + } + if (type === 'number') { + return `CAST(json_extract(data, '$.${escapeString(field)}') AS NUMERIC)`; + } + return `json_extract(data, '$.${escapeString(field)}')`; + }; + + const safeFieldNames = keyEntries.map(([ k ]) => k.replace(/[^a-zA-Z0-9]/g, '_')).join('_'); + const indexName = options.name + ? sanitizeIndexName(options.name) + : `idx_${this._tableName}_${safeFieldNames}`.substring(0, 63); + + const mongoName = options.name || keyEntries.map(([ k, v ]) => `${k}_${v}`).join('_'); + + this._indexes.set(indexName, { + keys, + options, + mongoName + }); + + const tableName = this._quotedTableName(); + const escapedIndexName = escapeIdentifier(indexName); + + // Build WHERE clause for sparse indexes + let whereClause = ''; + if (options.sparse) { + const sparseConditions = keyEntries.map(([ field ]) => { + if (field === '_id') { + return '_id IS NOT NULL'; + } + return `json_type(data, '$.${escapeString(field)}') IS NOT NULL`; + }); + whereClause = ` WHERE ${sparseConditions.join(' AND ')}`; + } + + // Handle text indexes — create an FTS5 virtual table + const hasTextIndex = keyEntries.some(([ , v ]) => v === 'text'); + if (hasTextIndex) { + let textFields = keyEntries.filter(([ , v ]) => v === 'text').map(([ k ]) => k); + // MongoDB dumps store text indexes as { _fts: 'text', _ftsx: 1 } + // The real field names are in options.weights + if (textFields.length === 1 && textFields[0] === '_fts' && options.weights) { + textFields = Object.keys(options.weights); + } + this._textFields = textFields; + + const fts = this._ftsTableName(); + const cols = textFields.map(f => `"${escapeIdentifier(f)}"`).join(', '); + + // Content-less FTS5 table — we manage inserts/deletes manually. + // content='' avoids duplicating data, contentless_delete=1 allows + // DELETE operations on the virtual table. + this._db._sqlite.exec( + `CREATE VIRTUAL TABLE IF NOT EXISTS ${fts} USING fts5(${cols}, content='', contentless_delete=1)` + ); + + // Clear any stale FTS entries (important during DB restore where + // deleteMany runs before createIndex sets _textFields, leaving + // orphaned FTS entries that cause incorrect search results) + this._db._sqlite.exec(`DELETE FROM ${fts}`); + + // Backfill existing documents into the FTS5 table + const rows = this._db._sqlite.prepare( + `SELECT rowid, _id, data FROM ${tableName}` + ).all(); + if (rows.length > 0) { + const insertFts = this._db._sqlite.prepare( + `INSERT OR REPLACE INTO ${fts} (rowid, ${cols}) VALUES (?, ${textFields.map(() => '?').join(', ')})` + ); + const backfill = this._db._sqlite.transaction(() => { + for (const row of rows) { + const doc = deserializeDocument(row.data, row._id); + const values = this._extractFtsValues(doc); + insertFts.run(row.rowid, ...values); + } + }); + backfill(); + } + + return indexName; + } + + // Handle unique constraint + if (options.unique) { + const indexExprs = keyEntries.map(([ field ]) => { + return field === '_id' ? '_id' : `(${buildIndexPath(field, indexType)})`; + }); + + this._db._sqlite.exec(` + CREATE UNIQUE INDEX IF NOT EXISTS "${escapedIndexName}" + ON ${tableName} (${indexExprs.join(', ')})${whereClause} + `); + return indexName; + } + + // Handle regular indexes + const indexExprs = keyEntries.map(([ field, direction ]) => { + if (field === '_id') { + return `_id ${direction === -1 ? 'DESC' : 'ASC'}`; + } + return `(${buildIndexPath(field, indexType)}) ${direction === -1 ? 'DESC' : 'ASC'}`; + }); + + this._db._sqlite.exec(` + CREATE INDEX IF NOT EXISTS "${escapedIndexName}" + ON ${tableName} (${indexExprs.join(', ')})${whereClause} + `); + + return indexName; + } + + async ensureIndex(keys, options) { + return this.createIndex(keys, options); + } + + async dropIndex(indexName) { + let pgName = null; + for (const [ pgKey, meta ] of this._indexes.entries()) { + if (meta.mongoName === indexName) { + pgName = pgKey; + break; + } + } + if (!pgName) { + // Same sanitization rules as createIndex — tolerate MongoDB-style + // names (e.g. "slug.1") that would fail strict identifier checks. + pgName = sanitizeIndexName(indexName); + } + this._indexes.delete(pgName); + const escapedIndexName = escapeIdentifier(pgName); + this._db._sqlite.exec(`DROP INDEX IF EXISTS "${escapedIndexName}"`); + } + + async indexes() { + this._ensureTable(); + + const rows = this._db._sqlite.prepare( + 'SELECT name, sql FROM sqlite_master WHERE type = \'index\' AND tbl_name = ?' + ).all(this._tableName); + + const indexes = [ { + name: '_id_', + key: { _id: 1 }, + unique: true + } ]; + + const seen = new Set(); + for (const row of rows) { + // Skip auto-created indexes (like the PRIMARY KEY index) + if (!row.sql) { + continue; + } + + seen.add(row.name); + const storedIndex = this._indexes.get(row.name); + if (storedIndex) { + indexes.push({ + name: storedIndex.mongoName || row.name, + key: storedIndex.keys, + unique: storedIndex.options.unique || false, + ...(storedIndex.options.sparse ? { sparse: true } : {}), + ...(storedIndex.options.type ? { type: storedIndex.options.type } : {}) + }); + } else { + indexes.push({ + name: row.name, + ...parseIndexDef(row.sql) + }); + } + } + + // Include indexes tracked in _indexes but not in sqlite_master + // (e.g. FTS5 virtual table text indexes) + for (const [ name, storedIndex ] of this._indexes) { + if (!seen.has(name)) { + indexes.push({ + name: storedIndex.mongoName || name, + key: storedIndex.keys, + unique: storedIndex.options.unique || false, + ...(storedIndex.options.sparse ? { sparse: true } : {}), + ...(storedIndex.options.type ? { type: storedIndex.options.type } : {}) + }); + } + } + + return indexes; + } + + async indexInformation() { + const indexes = await this.indexes(); + const info = {}; + for (const idx of indexes) { + info[idx.name] = Object.entries(idx.key).map(([ k, v ]) => [ k, v ]); + } + return info; + } + + async insert(docs) { + if (Array.isArray(docs)) { + return this.insertMany(docs); + } + return this.insertOne(docs); + } + + initializeUnorderedBulkOp() { + const collection = this; + const operations = []; + + return { + find(query) { + return { + updateOne(update) { + operations.push({ + updateOne: { + filter: query, + update + } + }); + }, + update(update) { + operations.push({ + updateMany: { + filter: query, + update + } + }); + }, + upsert() { + return { + updateOne(update) { + operations.push({ + updateOne: { + filter: query, + update, + upsert: true + } + }); + }, + update(update) { + operations.push({ + updateMany: { + filter: query, + update, + upsert: true + } + }); + }, + replaceOne(doc) { + operations.push({ + replaceOne: { + filter: query, + replacement: doc, + upsert: true + } + }); + } + }; + }, + deleteOne() { + operations.push({ deleteOne: { filter: query } }); + }, + delete() { + operations.push({ deleteMany: { filter: query } }); + } + }; + }, + async execute() { + return collection.bulkWrite(operations); + } + }; + } + + async drop() { + if (this._textFields) { + const fts = this._ftsTableName(); + this._db._sqlite.exec(`DROP TABLE IF EXISTS ${fts}`); + this._textFields = null; + } + const tableName = this._quotedTableName(); + this._db._sqlite.exec(`DROP TABLE IF EXISTS ${tableName}`); + this._initialized = false; + this._indexes.clear(); + } + + async rename(newName) { + const oldName = this._name; + const newCollName = validateTableName(newName); + const tableName = this._quotedTableName(); + const escapedNewTableName = escapeIdentifier(newCollName); + + // Rename FTS5 table first if it exists + if (this._textFields) { + const oldFts = this._ftsTableName(); + const newFtsName = escapeIdentifier(newCollName + '_fts'); + this._db._sqlite.exec(`ALTER TABLE ${oldFts} RENAME TO "${newFtsName}"`); + } + + this._db._sqlite.exec(`ALTER TABLE ${tableName} RENAME TO "${escapedNewTableName}"`); + + this._tableName = newCollName; + this._name = newName; + + this._db._collections.delete(oldName); + this._db._collections.set(newName, this); + } +} + +// ============================================================================= +// Database Implementation +// ============================================================================= + +class SqliteDb { + constructor(client, name, sqliteInstance) { + this._client = client; + this._sqlite = sqliteInstance; + this._name = name; + this.databaseName = name; + this._collections = new Map(); + } + + collection(name) { + if (!this._collections.has(name)) { + this._collections.set(name, new SqliteCollection(this, name)); + } + return this._collections.get(name); + } + + async createCollection(name) { + const col = this.collection(name); + col._ensureTable(); + return col; + } + + admin() { + const client = this._client; + return { + async listDatabases() { + // List all sibling database files in the directory, + // matching the extension of the original database + const dir = path.dirname(client._dbPath); + const ext = client._ext; + const fs = require('fs'); + let files; + try { + files = fs.readdirSync(dir); + } catch (e) { + return { databases: [] }; + } + const databases = files + .filter(f => f.endsWith(ext)) + .map(f => ({ name: f.slice(0, -ext.length) })); + return { databases }; + } + }; + } + + async dropDatabase() { + // Drop all tables + const tables = this._sqlite.prepare( + 'SELECT name FROM sqlite_master WHERE type = \'table\' AND name NOT LIKE \'sqlite_%\'' + ).all(); + for (const table of tables) { + this._sqlite.exec(`DROP TABLE IF EXISTS "${escapeIdentifier(table.name)}"`); + } + // Also drop all indexes + const indexes = this._sqlite.prepare( + 'SELECT name FROM sqlite_master WHERE type = \'index\' AND name NOT LIKE \'sqlite_%\'' + ).all(); + for (const idx of indexes) { + this._sqlite.exec(`DROP INDEX IF EXISTS "${escapeIdentifier(idx.name)}"`); + } + this._collections.clear(); + } + + async collections() { + const list = await this.listCollections().toArray(); + return list.map(entry => this.collection(entry.name)); + } + + listCollections() { + const self = this; + return { + async toArray() { + const rows = self._sqlite.prepare( + 'SELECT name FROM sqlite_master WHERE type = \'table\' AND name NOT LIKE \'sqlite_%\' AND name NOT LIKE \'%\\_fts%\' ESCAPE \'\\\'' + ).all(); + return rows.map(row => ({ name: row.name })); + } + }; + } +} + +// ============================================================================= +// Client Implementation +// ============================================================================= + +class SqliteClient { + constructor(sqliteInstance, dbPath, defaultDbName) { + this._sqlite = sqliteInstance; + this._dbPath = dbPath; + this._defaultDbName = defaultDbName; + this._ext = path.extname(dbPath) || '.sqlite'; + this._databases = new Map(); + this._siblingDbs = new Map(); + } + + db(name) { + if (!name) { + if (!this._databases.has(this._defaultDbName)) { + this._databases.set( + this._defaultDbName, + new SqliteDb(this, this._defaultDbName, this._sqlite) + ); + } + return this._databases.get(this._defaultDbName); + } + + // For sibling databases, open a separate file in the same directory, + // using the same extension as the original database + if (!this._databases.has(name)) { + const dir = path.dirname(this._dbPath); + const siblingPath = path.join(dir, `${name}${this._ext}`); + let siblingDb; + if (this._siblingDbs.has(siblingPath)) { + siblingDb = this._siblingDbs.get(siblingPath); + } else { + siblingDb = new Database(siblingPath); + _registerFunctions(siblingDb); + this._siblingDbs.set(siblingPath, siblingDb); + } + this._databases.set(name, new SqliteDb(this, name, siblingDb)); + } + return this._databases.get(name); + } + + async close() { + if (!this._closed) { + this._closed = true; + this._sqlite.close(); + for (const [ , db ] of this._siblingDbs) { + db.close(); + } + } + } +} + +// ============================================================================= +// Register custom regexp/regexp_i functions +// ============================================================================= + +function _registerFunctions(db) { + db.function('regexp', { deterministic: true }, (pattern, value) => { + if (value === null || value === undefined) { + return 0; + } + try { + return new RegExp(pattern).test(String(value)) ? 1 : 0; + } catch (e) { + return 0; + } + }); + + db.function('regexp_i', { deterministic: true }, (pattern, value) => { + if (value === null || value === undefined) { + return 0; + } + try { + return new RegExp(pattern, 'i').test(String(value)) ? 1 : 0; + } catch (e) { + return 0; + } + }); +} + +// ============================================================================= +// Module Export +// ============================================================================= + +module.exports = { + name: 'sqlite', + protocols: [ 'sqlite' ], + + async connect(uri, options = {}) { + // Parse URI: sqlite:///path/to/file.sqlite or sqlite://path/to/file.db + const url = new URL(uri); + let dbPath; + if (url.hostname) { + // sqlite://relative/path/to/file.sqlite + dbPath = url.hostname + url.pathname; + } else { + // sqlite:///absolute/path/to/file.sqlite + dbPath = url.pathname; + } + + // Reject in-memory / empty paths. The adapter does not support + // throwaway in-memory sqlite databases: an ApostropheCMS site needs + // a persistent store, and transient in-memory URIs would also + // interact poorly with multi-connection patterns (each new + // connection would get a fresh, empty database). + if (!dbPath || dbPath === ':memory:' || dbPath.endsWith('/:memory:')) { + throw new Error( + `sqlite adapter does not support in-memory databases (got "${uri}"). ` + + 'Provide a file path, e.g. sqlite:///var/lib/mysite/data.db' + ); + } + + // Ensure directory exists + const dir = path.dirname(dbPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const db = new Database(dbPath); + + // Enable WAL mode for better concurrent access + db.pragma('journal_mode = WAL'); + // Set busy timeout to wait up to 5 seconds for locks + db.pragma('busy_timeout = 5000'); + + // Register custom functions + _registerFunctions(db); + + const dbName = path.basename(dbPath, path.extname(dbPath)); + + return new SqliteClient(db, dbPath, dbName); + } +}; diff --git a/packages/db-connect/bin/apos-db-dump.js b/packages/db-connect/bin/apos-db-dump.js new file mode 100755 index 0000000000..6a9511e0c1 --- /dev/null +++ b/packages/db-connect/bin/apos-db-dump.js @@ -0,0 +1,42 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const dump = require('../lib/dump'); + +main().then(() => { + process.exit(0); +}).catch(err => { + console.error(err.message || err); + process.exit(1); +}); + +async function main() { + const args = process.argv.slice(2); + const uri = args.find(a => !a.startsWith('--')); + const outputArg = args.find(a => a.startsWith('--output=')); + const output = outputArg ? outputArg.split('=')[1] : null; + + if (!uri) { + throw new Error('Usage: apos-db-dump [--output=filename]'); + } + + // Stream NDJSON lines directly to the sink so a large dump never sits + // fully in memory. + const sink = output + ? fs.createWriteStream(output) + : process.stdout; + + try { + for await (const line of dump(uri)) { + if (!sink.write(line + '\n')) { + await new Promise(resolve => sink.once('drain', resolve)); + } + } + } finally { + if (output) { + await new Promise((resolve, reject) => { + sink.end(err => err ? reject(err) : resolve()); + }); + } + } +} diff --git a/packages/db-connect/bin/apos-db-restore.js b/packages/db-connect/bin/apos-db-restore.js new file mode 100755 index 0000000000..10abad25ce --- /dev/null +++ b/packages/db-connect/bin/apos-db-restore.js @@ -0,0 +1,30 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const restore = require('../lib/restore'); + +main().then(() => { + process.exit(0); +}).catch(err => { + console.error(err.message || err); + process.exit(1); +}); + +async function main() { + const args = process.argv.slice(2); + const uri = args.find(a => !a.startsWith('--')); + const inputArg = args.find(a => a.startsWith('--input=')); + const input = inputArg ? inputArg.split('=')[1] : null; + + if (!uri) { + throw new Error('Usage: apos-db-restore [--input=filename]'); + } + + // Stream the dump into restore() line-by-line so a large dump never + // sits fully in memory. restore() accepts any Readable stream. + const source = input + ? fs.createReadStream(input, { encoding: 'utf8' }) + : process.stdin; + + await restore(uri, source); +} diff --git a/packages/db-connect/claude-tools/logs/dump-mongodb.log b/packages/db-connect/claude-tools/logs/dump-mongodb.log new file mode 100644 index 0000000000..f144dba279 --- /dev/null +++ b/packages/db-connect/claude-tools/logs/dump-mongodb.log @@ -0,0 +1,11 @@ + + + dump/restore programmatic API (mongodb) + ✔ dump yields an async iterable of NDJSON lines (73ms) + ✔ should restore a database from a dump stream (69ms) + ✔ should copy a database via copyDatabase() (76ms) + ✔ should produce independent databases after copy + + + 4 passing (300ms) + diff --git a/packages/db-connect/claude-tools/logs/dump-multipostgres.log b/packages/db-connect/claude-tools/logs/dump-multipostgres.log new file mode 100644 index 0000000000..9dc7b86271 --- /dev/null +++ b/packages/db-connect/claude-tools/logs/dump-multipostgres.log @@ -0,0 +1,11 @@ + + + dump/restore programmatic API (multipostgres) + ✔ dump yields an async iterable of NDJSON lines (68ms) + ✔ should restore a database from a dump stream (78ms) + ✔ should copy a database via copyDatabase() (98ms) + ✔ should produce independent databases after copy (64ms) + + + 4 passing (426ms) + diff --git a/packages/db-connect/claude-tools/logs/dump-postgres.log b/packages/db-connect/claude-tools/logs/dump-postgres.log new file mode 100644 index 0000000000..5067056af0 --- /dev/null +++ b/packages/db-connect/claude-tools/logs/dump-postgres.log @@ -0,0 +1,11 @@ + + + dump/restore programmatic API (postgres) + ✔ dump yields an async iterable of NDJSON lines (67ms) + ✔ should restore a database from a dump stream (75ms) + ✔ should copy a database via copyDatabase() (95ms) + ✔ should produce independent databases after copy (62ms) + + + 4 passing (357ms) + diff --git a/packages/db-connect/claude-tools/logs/dump-sqlite.log b/packages/db-connect/claude-tools/logs/dump-sqlite.log new file mode 100644 index 0000000000..a3feb36382 --- /dev/null +++ b/packages/db-connect/claude-tools/logs/dump-sqlite.log @@ -0,0 +1,11 @@ + + + dump/restore programmatic API (sqlite) + ✔ dump yields an async iterable of NDJSON lines + ✔ should restore a database from a dump stream + ✔ should copy a database via copyDatabase() + ✔ should produce independent databases after copy + + + 4 passing (118ms) + diff --git a/packages/db-connect/docs/aggregation.md b/packages/db-connect/docs/aggregation.md new file mode 100644 index 0000000000..d426fb3ced --- /dev/null +++ b/packages/db-connect/docs/aggregation.md @@ -0,0 +1,118 @@ +# Aggregation + +The `aggregate` method runs a pipeline of stages that transform and summarize documents. It returns a cursor with a `toArray()` method. + +```js +const results = await collection.aggregate([ + { $match: { type: 'order' } }, + { $group: { + _id: '$status', + total: { $sum: '$amount' }, + count: { $sum: 1 } + }}, + { $sort: { total: -1 } } +]).toArray(); +``` + +When `$match` is the first pipeline stage, its query is passed to the database for filtering — only matching documents are loaded into memory. All subsequent stages are processed in memory. Place `$match` first whenever possible to reduce the working set. + +## Stages + +### $match + +Filters documents using standard [query operators](./queries.md). When `$match` is the first stage, filtering happens in the database — only matching documents are loaded into memory. Later `$match` stages filter in memory. Place `$match` first whenever possible. + +```js +{ $match: { status: 'active', price: { $gt: 10 } } } +``` + +### $group + +Groups documents by a key and computes aggregate values. The `_id` field specifies the grouping expression — use a field path (prefixed with `$`) or `null` to group all documents together. + +```js +{ $group: { + _id: '$category', + totalRevenue: { $sum: '$price' }, + averagePrice: { $avg: '$price' }, + count: { $sum: 1 } +}} +``` + +#### Accumulators + +| Accumulator | Description | +|-------------|-------------| +| `$sum` | Sums numeric values. Use `{ $sum: 1 }` to count documents in each group. Use `{ $sum: '$fieldName' }` to sum a field's values. | +| `$avg` | Computes the arithmetic mean of numeric values. | +| `$first` | Returns the value from the *first* document in each group. Order depends on preceding `$sort` stages. | +| `$last` | Returns the value from the *last* document in each group. | + +> **Not supported:** `$min`, `$max`, `$push`, `$addToSet`. + +### $project + +Reshapes documents by including or excluding fields. Uses the same projection syntax as [`find().project()`](./collections.md). + +```js +{ $project: { name: 1, email: 1, _id: 0 } } +``` + +### $unwind + +Deconstructs an array field, outputting one document per array element. Each output document replaces the array with a single element. + +```js +{ $unwind: '$tags' } + +// Object form also supported: +{ $unwind: { path: '$tags' } } +``` + +Given `{ title: 'Post', tags: ['a', 'b'] }`, `$unwind` produces: + +```js +{ title: 'Post', tags: 'a' } +{ title: 'Post', tags: 'b' } +``` + +### $sort + +Sorts documents. `1` for ascending, `-1` for descending. + +```js +{ $sort: { createdAt: -1 } } +{ $sort: { category: 1, price: -1 } } +``` + +### $limit + +Limits the number of documents passed to the next stage. + +```js +{ $limit: 10 } +``` + +### $skip + +Skips the specified number of documents. + +```js +{ $skip: 20 } +``` + +## Example: Top Categories by Revenue + +```js +const topCategories = await orders.aggregate([ + { $match: { status: 'completed' } }, + { $group: { + _id: '$category', + revenue: { $sum: '$total' }, + orders: { $sum: 1 }, + avgOrder: { $avg: '$total' } + }}, + { $sort: { revenue: -1 } }, + { $limit: 5 } +]).toArray(); +``` diff --git a/packages/db-connect/docs/collections.md b/packages/db-connect/docs/collections.md new file mode 100644 index 0000000000..3a84ac5aa3 --- /dev/null +++ b/packages/db-connect/docs/collections.md @@ -0,0 +1,293 @@ +# Collection Methods + +All collection methods are accessed from a collection object: + +```js +const db = client.db(); +const articles = db.collection('articles'); +``` + +## insertOne(doc) + +Inserts a single document. If `_id` is not provided, a random 24-character hex string is generated. If an ObjectId is passed as `_id`, it is converted to its hex string representation. `_id` is always a string. + +Throws a duplicate key error (code `11000`) if the `_id` or any unique index constraint is violated. + +```js +const result = await collection.insertOne({ title: 'Hello' }); +``` + +Returns: + +```js +{ + acknowledged: true, + insertedId: '64a1b2c3d4e5f6a7b8c9d0e1', + insertedCount: 1, + ops: [{ _id: '...', title: 'Hello' }], + result: { ok: 1 } +} +``` + +## insertMany(docs) + +Inserts an array of documents. Documents are inserted sequentially; insertion stops on the first error. + +```js +const result = await collection.insertMany([ + { title: 'First' }, + { title: 'Second' } +]); +``` + +Returns: + +```js +{ + acknowledged: true, + insertedCount: 2, + insertedIds: { 0: 'id1', 1: 'id2' }, + result: { ok: 1 } +} +``` + +`insertedIds` is a map of array index to `_id`. + +## findOne(query, options) + +Returns the first matching document, or `null` if no documents match. + +```js +const doc = await collection.findOne({ slug: 'hello' }); + +// With projection +const doc = await collection.findOne( + { slug: 'hello' }, + { projection: { title: 1, slug: 1 } } +); +``` + +**Options:** + +- `projection` — field inclusion or exclusion specification. `{ title: 1, slug: 1 }` returns only those fields plus `_id`. `{ body: 0 }` returns everything except `body`. `_id` is always included unless explicitly excluded with `{ _id: 0 }`. + +## find(query) + +Returns a cursor for iterating over matching documents. Cursor methods are chainable and must be called before `toArray()`. + +```js +const docs = await collection.find({ type: 'article' }) + .sort({ createdAt: -1 }) + .limit(10) + .skip(20) + .project({ title: 1, slug: 1 }) + .toArray(); +``` + +### Cursor Methods + +| Method | Description | +|--------|-------------| +| `.sort(spec)` | Sort order. `{ field: 1 }` for ascending, `{ field: -1 }` for descending. Multiple fields are supported. | +| `.limit(n)` | Maximum number of documents to return. `0` means no limit. | +| `.skip(n)` | Number of documents to skip before returning results. | +| `.project(spec)` | Field projection, same as the `projection` option on `findOne`. | +| `.clone()` | Returns an independent copy of the cursor. | +| `.toArray()` | Executes the query and returns all matching documents as an array. | +| `.next()` | Returns the next document, or `null` when exhausted. Also accepts a `(err, doc)` callback. | +| `.count()` | Returns the total count of matching documents. Ignores `limit` and `skip`. | +| `.addCursorFlag()` | No-op, provided for MongoDB driver compatibility. | + +### Async Iteration + +Cursors support `for await...of`: + +```js +for await (const doc of collection.find({ type: 'article' })) { + // process each document +} +``` + +## updateOne(query, update, options) + +Updates the first document matching the query using [update operators](./updates.md). + +```js +await collection.updateOne( + { slug: 'hello' }, + { $set: { title: 'Updated' } } +); + +// With upsert +await collection.updateOne( + { slug: 'hello' }, + { $set: { title: 'Upserted' } }, + { upsert: true } +); +``` + +**Options:** + +- `upsert` — if `true` and no document matches, inserts a new document. Fields from the query are merged into the new document along with the update operations. + +Returns: + +```js +{ + acknowledged: true, + matchedCount: 1, // 0 if no match (and no upsert) + modifiedCount: 1, // 0 if no changes made + upsertedId: null, // set to the new _id when upserted + upsertedCount: 0, // 1 when upserted + result: { nModified: 1, n: 1 } +} +``` + +When upserting, `matchedCount` is `0`, `upsertedCount` is `1`, and `modifiedCount` is `0`. + +## updateMany(query, update, options) + +Updates all documents matching the query. Same options and return structure as `updateOne`, but counts reflect all matched documents. + +```js +await collection.updateMany( + { status: 'draft' }, + { $set: { status: 'published' } } +); +``` + +## replaceOne(query, replacement, options) + +Replaces the entire document (except `_id`) with the given replacement object. This is not an update operation — the replacement document should not contain update operators. + +Supports `upsert: true`. + +```js +await collection.replaceOne( + { slug: 'hello' }, + { slug: 'hello', title: 'Replaced', body: 'New content' } +); +``` + +Returns the same structure as `updateOne`. + +## deleteOne(query) + +Deletes the first document matching the query. + +```js +const result = await collection.deleteOne({ slug: 'hello' }); +``` + +Returns: + +```js +{ + acknowledged: true, + deletedCount: 1, // 0 if no match + result: { ok: 1 } +} +``` + +## deleteMany(query) + +Deletes all documents matching the query. + +```js +const result = await collection.deleteMany({ status: 'archived' }); +// result.deletedCount — number of documents removed +``` + +## findOneAndUpdate(query, update, options) + +Finds a document, updates it, and returns it. Note: this operation uses a read-modify-write pattern and is **not atomic**. See the [Atomicity](../README.md#atomicity) section in the README. + +```js +const result = await collection.findOneAndUpdate( + { slug: 'hello' }, + { $inc: { views: 1 } }, + { returnDocument: 'after' } +); +// result.value — the document +``` + +**Options:** + +- `returnDocument` — `'before'` (default) returns the document as it was before the update. `'after'` returns the updated document. +- `upsert` — if `true`, inserts a new document when no match is found. + +## findOneAndReplace(query, replacement, options) + +Like `findOneAndUpdate`, but replaces the entire document. Same options. + +## countDocuments(query) + +Returns the count of documents matching the query. + +```js +const count = await collection.countDocuments({ type: 'article' }); +``` + +## distinct(field, query) + +Returns an array of distinct values for a field across all matching documents. + +Array fields are automatically flattened — if some documents have a scalar value and others have an array, all values are collected and deduplicated. + +```js +const types = await collection.distinct('type'); +const tags = await collection.distinct('tags', { published: true }); + +// If documents are: { tags: 'a' }, { tags: ['a', 'b'] } +// Result: ['a', 'b'] +``` + +## bulkWrite(operations) + +Executes an array of write operations in sequence. + +```js +await collection.bulkWrite([ + { insertOne: { document: { title: 'New' } } }, + { updateOne: { filter: { slug: 'old' }, update: { $set: { title: 'Updated' } } } }, + { updateMany: { filter: { status: 'draft' }, update: { $set: { reviewed: true } } } }, + { replaceOne: { filter: { slug: 'replace-me' }, replacement: { slug: 'replace-me', title: 'Replaced' } } }, + { deleteOne: { filter: { slug: 'gone' } } }, + { deleteMany: { filter: { status: 'archived' } } } +]); +``` + +Returns: + +```js +{ + acknowledged: true, + insertedCount: 1, + modifiedCount: 2, + deletedCount: 3, + upsertedCount: 0, + insertedIds: { 0: 'new-id' }, + upsertedIds: {} +} +``` + +## aggregate(pipeline) + +Runs an aggregation pipeline. See [Aggregation](./aggregation.md) for supported stages and accumulators. + +```js +const results = await collection.aggregate([ + { $match: { type: 'order' } }, + { $group: { _id: '$status', total: { $sum: '$amount' } } } +]).toArray(); +``` + +## drop() + +Drops the collection and all its data. + +## rename(newName) + +Renames the collection. + +See also: [Database and Client Methods](./database.md) for `client.db()`, `db.listCollections()`, `db.dropDatabase()`, and other database-level operations. diff --git a/packages/db-connect/docs/database.md b/packages/db-connect/docs/database.md new file mode 100644 index 0000000000..a9caaa2371 --- /dev/null +++ b/packages/db-connect/docs/database.md @@ -0,0 +1,94 @@ +# Database and Client Methods + +## Client + +`connect(uri)` returns a client object. The client manages the connection pool and provides access to databases. + +```js +const connect = require('@apostrophecms/db-connect'); + +const client = await connect('postgres://localhost:5432/mydb'); +const db = client.db(); +// ... +await client.close(); +``` + +### client.db(name) + +Returns a database object. When called without a name, returns the default database derived from the connection URI. + +When called with a name, the behavior depends on the adapter: + +**MongoDB:** Behaves as in the native driver — each name accesses a separate MongoDB database on the same server. + +**PostgreSQL (single mode, `postgres://`):** `db()` or `db('mydb')` (matching the database name from the connection URI) both return the default database. Passing a different name throws an error — there is no schema isolation in single mode. + +**PostgreSQL (multi-schema mode, `multipostgres://`):** Each name must be a full virtual database name in the form `realdb-schema`, where `realdb` matches the physical database name from the connection URI. The schema name is everything after the **last** hyphen — this accommodates database names that themselves contain hyphens (e.g., `my-shared-db-tenant1` uses real database `my-shared-db` and schema `tenant1`). For example, if the URI is `multipostgres://localhost:5432/shareddb-tenant1`, then `db('shareddb-tenant2')` accesses the `tenant2` schema. Names that don't start with `realdb-` are rejected. Schemas are created automatically on first use. + +**SQLite:** Each name opens a separate file in the same directory as the original database file, using the same extension. For example, if the original URI points to `data/myapp.sqlite`, then `db('other')` opens `data/other.sqlite`. This provides true separation — each named database has its own tables and data. + +### client.close() + +Closes the connection pool. After calling `close()`, no further database operations should be attempted. + +## Database + +### db.collection(name) + +Returns a collection object. The collection is created automatically on first use (no need to call `createCollection` first). See [Collection Methods](./collections.md) for all available operations. + +```js +const articles = db.collection('articles'); +``` + +### db.listCollections() + +Returns a cursor-like object with a `toArray()` method that lists all collections in the database. + +```js +const collections = await db.listCollections().toArray(); +// [{ name: 'articles' }, { name: 'users' }, ...] +``` + +### db.createCollection(name) + +Creates a collection explicitly. In practice this is rarely needed, since collections are created automatically when first accessed. + +```js +await db.createCollection('newCollection'); +``` + +### db.dropDatabase() + +Drops the entire database: + +- **MongoDB:** Drops the database. +- **PostgreSQL (single mode):** Drops all collection tables. +- **PostgreSQL (multi-schema mode):** Drops the schema and all its tables. +- **SQLite:** Drops all tables and indexes, but does not delete the database file. + +```js +await db.dropDatabase(); +``` + +### db.admin().listDatabases() + +Returns a list of databases. In `multipostgres` mode, this lists schemas as full virtual database names. + +```js +const { databases } = await db.admin().listDatabases(); +// MongoDB: [{ name: 'mydb' }, { name: 'otherdb' }, ...] +// multipostgres: [{ name: 'shareddb-tenant1' }, { name: 'shareddb-tenant2' }, ...] +``` + +## Document Size Limits + +Each adapter has a different maximum document size. For portable applications, assume MongoDB's 16 MB cap applies everywhere — both PostgreSQL and SQLite accept considerably larger documents, but leaning on those higher ceilings makes the data non-portable between adapters and produces unpredictable query performance. + +| Adapter | Per-document limit | +|--------------|------------------------------------------------------------| +| MongoDB | 16 MB (BSON `maxBsonObjectSize`) | +| PostgreSQL | JSONB field up to ~255 MB after TOAST compression (practical limit well under 1 GB) | +| SQLite | Row/TEXT column capped by `SQLITE_MAX_LENGTH` (default 1 GB); `JSON1` parser also has internal limits | + +Exceeding the adapter-native limit produces an adapter-native error (not a portable one). Keep documents well under 16 MB to stay within the MongoDB cap and to avoid TOAST storage for most rows in PostgreSQL. diff --git a/packages/db-connect/docs/dump-restore.md b/packages/db-connect/docs/dump-restore.md new file mode 100644 index 0000000000..ab17cf018e --- /dev/null +++ b/packages/db-connect/docs/dump-restore.md @@ -0,0 +1,178 @@ +# Dump and Restore + +`@apostrophecms/db-connect` ships CLI tools and a programmatic API for exporting and importing databases in a portable JSONL format. Because the format is adapter-agnostic, you can dump from one database type and restore to another. + +## CLI Tools + +### Running the commands + +Any ApostropheCMS project has `@apostrophecms/db-connect` installed as a transitive dependency, so the `apos-db-dump` and `apos-db-restore` binaries are present in `node_modules/.bin` of every Apostrophe project. There are three common ways to invoke them: + +**Project-local with `npx` (recommended for one-off use):** + +```bash +cd /path/to/your/apostrophe/project +npx apos-db-dump mongodb://localhost:27017/mydb --output=backup.jsonl +npx apos-db-restore postgres://localhost:5432/mydb --input=backup.jsonl +``` + +`npx` finds the binary in the project's `node_modules/.bin`, so no global installation is needed. + +**Project-local via npm scripts:** + +Add entries to your project's `package.json`: + +```json +{ + "scripts": { + "db:dump": "apos-db-dump", + "db:restore": "apos-db-restore" + } +} +``` + +Then run: + +```bash +npm run db:dump -- mongodb://localhost:27017/mydb --output=backup.jsonl +``` + +**Globally installed (for operators who manage many projects):** + +```bash +npm install -g @apostrophecms/db-connect +apos-db-dump mongodb://localhost:27017/mydb --output=backup.jsonl +``` + +A global install puts `apos-db-dump` and `apos-db-restore` directly on your `PATH`. The global copy is independent of any particular project, so use the same major version you have installed in your projects to ensure the JSONL format matches. + +The examples in the rest of this document drop the `npx` prefix for readability. Prepend `npx` (or use one of the other invocation styles above) according to how you've set things up. + +### apos-db-dump + +```bash +# Dump to a file +apos-db-dump postgres://localhost:5432/mydb --output=backup.jsonl + +# Dump to stdout +apos-db-dump sqlite:///path/to/db.sqlite +``` + +### apos-db-restore + +```bash +# Restore from a file +apos-db-restore postgres://localhost:5432/mydb --input=backup.jsonl + +# Restore from stdin +cat backup.jsonl | apos-db-restore sqlite:///path/to/newdb.sqlite +``` + +### Cross-Database Migration + +Pipe `dump` output directly into `restore` to migrate between backends: + +```bash +# MongoDB to PostgreSQL +apos-db-dump mongodb://localhost:27017/mydb | apos-db-restore postgres://localhost:5432/mydb + +# PostgreSQL to SQLite +apos-db-dump postgres://localhost:5432/mydb | apos-db-restore sqlite:///path/to/local.db +``` + +### multipostgres URIs + +`multipostgres://` URIs are fully supported. The URI path is a complete virtual database name, split at the **last hyphen** into a real PostgreSQL database name and a schema name. This lets you dump or restore a single tenant's schema directly from the command line: + +```bash +# Dump just the tenant1 schema from the shareddb database +apos-db-dump multipostgres://localhost:5432/shareddb-tenant1 --output=tenant1.jsonl + +# Restore into a different tenant's schema +apos-db-restore multipostgres://localhost:5432/shareddb-tenant2 --input=tenant1.jsonl +``` + +The last-hyphen rule accommodates real database names that themselves contain hyphens — `multipostgres://localhost:5432/my-shared-db-tenant1` connects to the `my-shared-db` database and operates on the `tenant1` schema. A multipostgres URI with no hyphen in the path is rejected, since it cannot specify a complete virtual database. + +You can migrate between a single-database `postgres://` URI and a tenant schema by mixing protocols on either side of the pipe: + +```bash +# Copy a standalone database into a tenant schema +apos-db-dump postgres://localhost:5432/mydb | apos-db-restore multipostgres://localhost:5432/shareddb-tenant1 + +# Extract a tenant schema into its own standalone database +apos-db-dump multipostgres://localhost:5432/shareddb-tenant1 | apos-db-restore postgres://localhost:5432/tenant1db +``` + +## Programmatic API + +```js +const { dump, restore, copyDatabase } = require('@apostrophecms/db-connect'); +``` + +### dump(uri) + +Returns an **async iterable** that yields one JSONL record per line (no trailing newlines). The dump is produced incrementally so large databases never sit fully in memory. + +```js +for await (const line of dump('postgres://localhost:5432/mydb')) { + process.stdout.write(line + '\n'); +} +``` + +To collect the entire dump as a string (only safe for small databases): + +```js +const lines = []; +for await (const line of dump(uri)) lines.push(line); +const data = lines.join('\n') + '\n'; +``` + +Also accepts an already-connected `db` object instead of a URI string. + +### restore(uri, source) + +Imports a JSONL stream into the database. `source` can be any of: + +- an async iterable of JSONL lines (as produced by `dump()`) +- an iterable / array of JSONL lines +- a Node Readable stream (e.g. `process.stdin`, `fs.createReadStream(...)`) +- a single JSONL string — retained for convenience, not recommended for large dumps + +```js +// Async iterable +await restore('sqlite:///path/to/db.sqlite', dump(sourceUri)); + +// Readable stream +await restore('sqlite:///path/to/db.sqlite', fs.createReadStream('backup.jsonl')); + +// String (small dumps only) +await restore('sqlite:///path/to/db.sqlite', dataString); +``` + +Also accepts an already-connected `db` object. + +### copyDatabase(sourceUri, destUri) + +Copies all data directly from one database to another by piping `dump()` straight into `restore()`. No intermediate buffer — works regardless of database size. + +```js +await copyDatabase( + 'postgres://localhost:5432/source', + 'sqlite:///path/to/dest.db' +); +``` + +## JSONL Format + +The dump format is JSONL (one JSON object per line). Each collection begins with a header line containing the collection name and index definitions, followed by one line per document. + +``` +{"collection":"articles","indexes":[{"key":{"slug":1},"unique":true}]} +{"_id":"abc123","title":"Hello","slug":"hello"} +{"_id":"def456","title":"World","slug":"world"} +{"collection":"users","indexes":[]} +{"_id":"user1","email":"alice@example.com"} +``` + +On restore, indexes are recreated from the header. Index creation failures are fatal — a failed `createIndex` is a real problem (invalid index name, conflicting definition, backend incompatibility) and should surface to the caller rather than being silently tolerated. diff --git a/packages/db-connect/docs/indexes.md b/packages/db-connect/docs/indexes.md new file mode 100644 index 0000000000..130c89500d --- /dev/null +++ b/packages/db-connect/docs/indexes.md @@ -0,0 +1,114 @@ +# Indexes + +Indexes improve query performance and enforce uniqueness constraints. + +## createIndex(keys, options) + +Creates an index on one or more fields. Returns the index name as a string. + +```js +await collection.createIndex({ slug: 1 }); +``` + +Field values in the `keys` object: + +- `1` — ascending index +- `-1` — descending index +- `'text'` — full-text search index + +### Options + +| Option | Description | +|--------|-------------| +| `name` | Custom index name. If omitted, a name is generated from the field names (e.g., `slug_1`). | +| `unique` | If `true`, enforces a uniqueness constraint. Inserting a duplicate value throws a duplicate key error (code `11000`). | +| `sparse` | If `true`, only indexes documents where the indexed field exists. Documents without the field are omitted from the index. | +| `type` | Index value type: `'number'`, `'date'`, or omitted for the default text-based index. See below. | + +### Compound Indexes + +Pass multiple fields to create a compound index: + +```js +await collection.createIndex({ type: 1, createdAt: -1 }); +await collection.createIndex({ email: 1, tenant: 1 }, { unique: true }); +``` + +## Index Types + +By default, indexed values are extracted from the JSON document as text. This works correctly for equality checks (`$eq`, `$in`) and pattern matching (`$regex`), but **not** for range comparisons on numbers or dates, because text comparison produces incorrect ordering (e.g., `"9"` sorts after `"10"`). + +### Numeric Indexes + +Use `type: 'number'` when the field contains numeric values and you need range queries: + +```js +await collection.createIndex({ price: 1 }, { type: 'number' }); + +// Now these queries use the index correctly: +await collection.find({ price: { $gt: 10, $lte: 100 } }).toArray(); +``` + +Without the `number` type, the comparison is text-based and `{ $gt: 9 }` would not match `10`. + +### Date Indexes + +Use `type: 'date'` for fields that store dates. Dates are stored internally as `{ $date: "ISO-8601-string" }`, and the date index type extracts the ISO string for correct lexicographic ordering. + +```js +await collection.createIndex({ createdAt: 1 }, { type: 'date' }); + +await collection.find({ createdAt: { $gt: new Date('2024-01-01') } }).toArray(); +``` + +ISO-8601 strings sort correctly in chronological order because they are fixed-width and most-significant-digit-first. + +### Text Indexes + +Use `'text'` as the field value (not as an option) to create a full-text search index: + +```js +await collection.createIndex({ title: 'text', body: 'text' }); + +await collection.find({ $text: { $search: 'tutorial' } }).toArray(); +``` + +In PostgreSQL, this creates a GIN index using `to_tsvector`. In SQLite, this creates an FTS5 virtual table kept in sync with the collection, providing proper tokenized full-text search with BM25 relevance scoring. + +### Combined Options + +Index types can be combined with other options: + +```js +await collection.createIndex( + { publishedAt: 1 }, + { type: 'date', unique: true, sparse: true } +); +``` + +## Array Fields + +When a query matches a scalar value against an array field, the adapters handle this transparently. For example, `{ tags: 'news' }` matches a document where `tags` is `['news', 'featured']`. This works with all query operators including `$in`, `$all`, and comparison operators. + +Indexes on array fields work with this behavior — scalar lookups against array elements use the index. + +## dropIndex(name) + +Drops an index by name. + +```js +await collection.dropIndex('slug_1'); +``` + +## indexes() + +Returns an array of index metadata for the collection. + +```js +const idxs = await collection.indexes(); +// [ +// { name: '_id_', key: { _id: 1 }, unique: true }, +// { name: 'slug_1', key: { slug: 1 } }, +// { name: 'price_1', key: { price: 1 }, type: 'number' } +// ] +``` diff --git a/packages/db-connect/docs/queries.md b/packages/db-connect/docs/queries.md new file mode 100644 index 0000000000..4fffd40015 --- /dev/null +++ b/packages/db-connect/docs/queries.md @@ -0,0 +1,233 @@ +# Query Operators + +Queries filter documents by matching field values against conditions. A query is a plain object where each key is a field name (or a logical operator) and each value is a match condition. + +```js +// Implicit $eq — matches documents where status is 'active' +{ status: 'active' } + +// Explicit operator +{ age: { $gte: 18 } } + +// Multiple fields — all must match (implicit $and) +{ status: 'active', type: 'article' } +``` + +## Dot Notation + +Nested fields are accessed with dot notation: + +```js +{ 'address.city': 'Portland' } +{ 'meta.tags': 'featured' } +``` + +## Comparison Operators + +### $eq + +Matches documents where the field equals the given value. This is the default when a plain value is used. + +```js +// These are equivalent: +{ status: 'active' } +{ status: { $eq: 'active' } } +``` + +**Null matching:** `{ field: null }` matches documents where the field is `null` *and* documents where the field is missing entirely. + +**Array matching:** When the field contains an array, `$eq` matches if *any element* of the array equals the value. `{ tags: 'news' }` matches `{ tags: ['news', 'featured'] }`. + +### $ne + +Matches documents where the field does *not* equal the given value. Documents where the field is missing also match (since missing is not equal to any value). + +```js +{ status: { $ne: 'archived' } } +``` + +### $gt / $gte / $lt / $lte + +Range comparisons. For these to work correctly on numeric and date fields, ensure the appropriate [index type](./indexes.md) is set. + +```js +{ price: { $gt: 10 } } +{ price: { $gte: 10, $lte: 100 } } +{ createdAt: { $gt: new Date('2024-01-01') } } +``` + +**Numbers:** Values are cast to numeric type before comparison, so `9` sorts before `10` (not after, as it would in a text comparison). + +**Dates:** Dates stored as `{ $date: "ISO-8601-string" }` are compared as ISO strings, which sort correctly for chronological order. + +**Missing fields:** Documents where the field is missing do *not* match range operators. + +### $in + +Matches documents where the field's value is any of the values in the given array. + +```js +{ status: { $in: ['active', 'pending'] } } +``` + +An empty array matches nothing. If the array contains `null`, documents with missing fields also match. + +**Array fields:** If the document field is an array, matches if *any element* of the field is in the `$in` array. + +### $nin + +Matches documents where the field's value is *not* any of the values in the given array. + +```js +{ status: { $nin: ['archived', 'deleted'] } } +``` + +An empty array matches everything. If the array contains `null`, documents with missing fields are *excluded*. If the array does not contain `null`, documents with missing fields *are* included. + +## Logical Operators + +### $and + +Matches documents that satisfy *all* of the given subqueries. Useful when you need multiple conditions on the same field. + +```js +{ $and: [ + { price: { $gte: 10 } }, + { price: { $lte: 100 } } +]} +``` + +Multiple conditions on different fields in a single object are already an implicit `$and`: + +```js +// These are equivalent: +{ status: 'active', type: 'article' } +{ $and: [{ status: 'active' }, { type: 'article' }] } +``` + +### $or + +Matches documents that satisfy *at least one* of the given subqueries. + +```js +{ $or: [ + { status: 'active' }, + { featured: true } +]} +``` + +### $not + +Negates a single operator expression. Applied to a specific field. + +```js +{ price: { $not: { $gt: 100 } } } +``` + +## Element Operators + +### $exists + +Matches documents based on whether a field is present in the document. + +```js +{ optionalField: { $exists: true } } // field is present (even if null) +{ optionalField: { $exists: false } } // field is absent +``` + +Note: `$exists: true` matches documents where the field is present, including when the value is `null`. This differs from `{ field: { $ne: null } }`, which excludes both missing and null-valued fields. + +## Evaluation Operators + +### $regex + +Matches string fields against a regular expression. + +```js +{ name: { $regex: '^John' } } +{ name: { $regex: 'smith', $options: 'i' } } // case-insensitive +``` + +Use `$options: 'i'` for case-insensitive matching. + +**Array fields:** If the field contains an array of strings, matches if *any element* matches the pattern. + +A JavaScript `RegExp` object can also be passed directly as the value: + +```js +{ name: /^John/i } +``` + +### $text + +Full-text search. Requires a [text index](./indexes.md) on the collection. + +```js +{ $text: { $search: 'apostrophe tutorial' } } +``` + +The search uses OR semantics — a document matches if it contains *any* of the search terms. The text search examines the fields declared in the text index (for Apostrophe, typically `highSearchText`, `lowSearchText`, `title`, and `searchBoost`). + +**Ranking:** Results can be ordered by relevance using the MongoDB-compatible `{ $meta: 'textScore' }` sort, and the score can be projected as well: + +```js +collection + .find({ $text: { $search: 'apostrophe tutorial' } }) + .sort({ score: { $meta: 'textScore' } }) + .project({ score: { $meta: 'textScore' } }) + .toArray(); +``` + +Without a `$meta: 'textScore'` sort, results are returned in unspecified order — use `.sort()` if you need a deterministic order. + +**PostgreSQL:** Uses `to_tsvector`/`to_tsquery` with the `simple` dictionary, backed by a GIN index. Relevance scoring uses `ts_rank`. Special characters in search terms are stripped. + +**SQLite:** Uses an FTS5 virtual table kept in sync with the collection via triggers. Relevance scoring uses BM25. Tokenization is FTS5's default (Unicode-aware, case-insensitive) — there is no stemming or stop word removal, so word forms are matched literally. + +## Array Operators + +### $all + +Matches documents where an array field contains *all* of the specified values. + +```js +{ tags: { $all: ['javascript', 'tutorial'] } } +``` + +The array must contain every value in the `$all` array, but may contain additional values. + +### $size + +Matches documents where an array field has the specified number of elements. + +```js +{ tags: { $size: 2 } } // matches arrays with exactly 2 elements +{ tags: { $size: 0 } } // matches empty arrays +``` + +The value must be a non-negative integer. Only matches fields that are arrays — documents where the field is missing or not an array are excluded. + +## Array Field Matching + +Array fields are queried transparently. A scalar match against an array field succeeds if *any element* of the array matches: + +```js +// Document: { tags: ['news', 'featured'] } + +{ tags: 'news' } // matches (element equality) +{ tags: { $in: ['news', 'other'] } } // matches (element in array) +{ tags: { $regex: '^new' } } // matches (element matches pattern) +{ tags: { $all: ['news', 'featured'] } } // matches (all elements present) +``` + +### Array Matching and Index Usage + +In PostgreSQL and SQLite, there is a trade-off between transparent array matching and index usage. + +Simple equality queries like `{ tags: 'news' }` use PostgreSQL's `@>` (jsonb containment) operator, which transparently matches both scalar values and array elements. However, `@>` does not use standard btree indexes. + +Explicit operator queries like `{ tags: { $eq: 'news' } }` use strict equality (`=`), which can use btree indexes but does *not* match array elements — it only matches if the entire field value is the scalar `'news'`. + +The same applies to `$in`: simple `$in` uses containment (array-aware, not indexed by btree), while indexed fields are more efficient with direct equality lookups. + +For fields that are always arrays and need indexed queries, consider using `$all` or restructuring the query. For fields that are always scalars, `$eq` and btree indexes work as expected. diff --git a/packages/db-connect/docs/updates.md b/packages/db-connect/docs/updates.md new file mode 100644 index 0000000000..ed27a8bb1c --- /dev/null +++ b/packages/db-connect/docs/updates.md @@ -0,0 +1,99 @@ +# Update Operators + +Update operators modify documents in place. They are used with `updateOne`, `updateMany`, `findOneAndUpdate`, and in `bulkWrite` operations. + +Multiple operators can be combined in a single update: + +```js +await collection.updateOne( + { _id: 'abc' }, + { + $set: { title: 'Updated' }, + $inc: { revision: 1 }, + $currentDate: { updatedAt: true } + } +); +``` + +## $set + +Sets the value of one or more fields. Creates the field if it does not exist. Overwrites the existing value if it does. + +```js +{ $set: { title: 'New Title' } } +{ $set: { 'address.city': 'Portland', 'address.state': 'OR' } } +``` + +Dot notation creates nested structures automatically. An `undefined` value is stored as `null`. + +## $unset + +Removes one or more fields from the document. The value given for each field is ignored — any truthy value works. + +```js +{ $unset: { obsoleteField: 1 } } +{ $unset: { 'nested.field': '', otherField: true } } +``` + +## $inc + +Increments a numeric field by the given amount. If the field does not exist, it is created with the increment as its value (starting from `0`). + +```js +{ $inc: { views: 1 } } // increment by 1 +{ $inc: { stock: -3 } } // decrement by 3 +{ $inc: { views: 1, shares: 1 } } // increment multiple fields +``` + +Use negative values to decrement. + +**Upsert behavior:** When used with `upsert: true` and no document matches, `$inc` creates the field with the increment value (e.g., `{ $inc: { count: 5 } }` creates `count: 5`). + +## $push + +Appends a value to an array field. Creates the array if the field does not exist. + +```js +{ $push: { tags: 'new-tag' } } +{ $push: { comments: { author: 'Alice', text: 'Great post' } } } +``` + +Each `$push` appends a single element. To append to multiple array fields, include them all in one `$push`: + +```js +{ $push: { tags: 'new-tag', categories: 'tutorials' } } +``` + +> **Note:** The `$each` modifier is not supported. To add multiple elements, use multiple updates or a read-modify-write pattern. + +## $pull + +Removes *all* occurrences of a value from an array field. Uses deep equality, so it works with objects and nested arrays. + +```js +{ $pull: { tags: 'old-tag' } } +{ $pull: { items: { sku: 'ABC123' } } } // removes matching objects +``` + +## $addToSet + +Adds a value to an array field only if it is not already present. Uses deep equality for comparison. + +```js +{ $addToSet: { tags: 'unique-tag' } } +``` + +If the value already exists in the array, the array is not modified. + +> **Note:** The `$each` modifier is not supported. + +## $currentDate + +Sets a field to the current date as a JavaScript `Date` object. + +```js +{ $currentDate: { updatedAt: true } } +{ $currentDate: { updatedAt: { $type: 'date' } } } +``` + +Both forms are equivalent. Only the `'date'` type is supported. diff --git a/packages/db-connect/eslint.config.js b/packages/db-connect/eslint.config.js new file mode 100644 index 0000000000..e0cd3f580e --- /dev/null +++ b/packages/db-connect/eslint.config.js @@ -0,0 +1,6 @@ +const apostrophe = require('eslint-config-apostrophe').default; +const { defineConfig } = require('eslint/config'); + +module.exports = defineConfig([ + apostrophe +]); diff --git a/packages/db-connect/index.js b/packages/db-connect/index.js new file mode 100644 index 0000000000..0d75439434 --- /dev/null +++ b/packages/db-connect/index.js @@ -0,0 +1,44 @@ +const mongodbAdapter = require( + './adapters/mongodb.js' +); +const postgresAdapter = require( + './adapters/postgres.js' +); +const sqliteAdapter = require( + './adapters/sqlite.js' +); + +const connect = async function(uri, options = {}) { + // Build adapter map: built-ins first, then custom overrides + const named = new Map(); + for (const adapter of [ + mongodbAdapter, + postgresAdapter, + sqliteAdapter, + ...(options.adapters || []) + ]) { + named.set(adapter.name, adapter); + } + + // Match protocol from URI + const matches = uri.match(/^([^:]+):\/\//); + if (!matches) { + throw new Error(`Invalid database URI: ${uri}`); + } + const protocol = matches[1]; + + for (const adapter of named.values()) { + if (adapter.protocols.includes(protocol)) { + return adapter.connect(uri, options); + } + } + + throw new Error( + `No adapter found for protocol: ${protocol}` + ); +}; + +module.exports = connect; +module.exports.dump = require('./lib/dump'); +module.exports.restore = require('./lib/restore'); +module.exports.copyDatabase = require('./lib/copy-database'); diff --git a/packages/db-connect/lib/aggregation-cursor.js b/packages/db-connect/lib/aggregation-cursor.js new file mode 100644 index 0000000000..31d0cd7883 --- /dev/null +++ b/packages/db-connect/lib/aggregation-cursor.js @@ -0,0 +1,408 @@ +const { + getNestedField, + setNestedField, + deepEqual, + validateInteger +} = require('./shared'); + +class AggregationCursor { + constructor(collection, pipeline) { + this._collection = collection; + this._pipeline = pipeline; + } + + async toArray() { + let stages = this._pipeline; + let query = {}; + if (stages.length > 0 && stages[0].$match) { + query = stages[0].$match; + stages = stages.slice(1); + } + let docs = await this._collection.find(query).toArray(); + + for (const stage of stages) { + const [ op, value ] = Object.entries(stage)[0]; + + switch (op) { + case '$match': + docs = docs.filter(doc => matchesQuery(doc, value)); + break; + case '$group': + docs = this._processGroup(docs, value); + break; + case '$project': + docs = docs.map(doc => applyAggregateProject(doc, value)); + break; + case '$unwind': + docs = this._processUnwind(docs, value); + break; + case '$sort': + docs = this._processSort(docs, value); + break; + case '$limit': + docs = docs.slice(0, validateInteger(value, '$limit')); + break; + case '$skip': + docs = docs.slice(validateInteger(value, '$skip')); + break; + default: + throw new Error(`Unsupported aggregation stage: ${op}`); + } + } + + return docs; + } + + _processGroup(docs, groupSpec) { + const groups = new Map(); + const groupField = groupSpec._id; + + for (const doc of docs) { + let groupKey; + if (typeof groupField === 'string' && groupField.startsWith('$')) { + groupKey = getNestedField(doc, groupField.substring(1)); + } else if (groupField !== null && typeof groupField === 'object') { + groupKey = {}; + for (const [ k, v ] of Object.entries(groupField)) { + if (typeof v === 'string' && v.startsWith('$')) { + groupKey[k] = getNestedField(doc, v.substring(1)); + } else { + groupKey[k] = v; + } + } + } else { + groupKey = groupField; + } + + const keyStr = JSON.stringify(groupKey); + if (!groups.has(keyStr)) { + groups.set(keyStr, { + _id: groupKey, + docs: [] + }); + } + groups.get(keyStr).docs.push(doc); + } + + const results = []; + for (const [ , group ] of groups) { + const result = { _id: group._id }; + + for (const [ field, expr ] of Object.entries(groupSpec)) { + if (field === '_id') { + continue; + } + + if (expr.$sum) { + const sumField = expr.$sum; + if (typeof sumField === 'string' && sumField.startsWith('$')) { + result[field] = group.docs.reduce((sum, doc) => { + return sum + (getNestedField(doc, sumField.substring(1)) || 0); + }, 0); + } else if (typeof sumField === 'number') { + result[field] = group.docs.length * sumField; + } + } else if (expr.$avg) { + const avgField = expr.$avg.substring(1); + const values = group.docs + .map(doc => getNestedField(doc, avgField)) + .filter(v => v != null); + result[field] = values.length > 0 + ? values.reduce((a, b) => a + b, 0) / values.length + : null; + } else if (expr.$first) { + const firstField = expr.$first.substring(1); + result[field] = group.docs.length > 0 + ? getNestedField(group.docs[0], firstField) + : null; + } else if (expr.$last) { + const lastField = expr.$last.substring(1); + const last = group.docs[group.docs.length - 1]; + result[field] = group.docs.length > 0 + ? getNestedField(last, lastField) + : null; + } + } + + results.push(result); + } + + return results; + } + + _processUnwind(docs, field) { + const fieldName = field.startsWith('$') ? field.substring(1) : field; + const results = []; + + for (const doc of docs) { + const arr = getNestedField(doc, fieldName); + if (Array.isArray(arr)) { + for (const item of arr) { + const newDoc = JSON.parse(JSON.stringify(doc)); + setNestedField(newDoc, fieldName, item); + results.push(newDoc); + } + } else { + results.push(doc); + } + } + + return results; + } + + _processSort(docs, sortSpec) { + return docs.slice().sort((a, b) => { + for (const [ field, direction ] of Object.entries(sortSpec)) { + const aVal = getNestedField(a, field); + const bVal = getNestedField(b, field); + + if (aVal < bVal) { + return direction === -1 ? 1 : -1; + } + if (aVal > bVal) { + return direction === -1 ? -1 : 1; + } + } + return 0; + }); + } +} + +function applyAggregateProject(doc, spec) { + const result = {}; + let includeId = true; + for (const [ key, value ] of Object.entries(spec)) { + if (key === '_id') { + if (value === 0 || value === false) { + includeId = false; + } else if (value === 1 || value === true) { + includeId = true; + } + continue; + } + if (value === 0 || value === false) { + continue; + } + if (value === 1 || value === true) { + const v = getNestedField(doc, key); + if (v !== undefined) { + setNestedField(result, key, v); + } + } else if (typeof value === 'string' && value.startsWith('$')) { + const refPath = value.slice(1); + const v = getNestedField(doc, refPath); + if (v !== undefined) { + setNestedField(result, key, v); + } + } else { + setNestedField(result, key, value); + } + } + if (includeId && doc._id !== undefined) { + result._id = doc._id; + } + return result; +} + +// Operators supported inside a field's operator object, e.g. +// { field: { $gt: 3, $lt: 10 } }. Kept in sync with the SQL adapters' +// buildOperatorClause — any operator the main query path supports should +// match here, and unknown operators throw rather than silently succeeding. +const FIELD_OPERATORS = new Set([ + '$eq', '$ne', '$gt', '$gte', '$lt', '$lte', + '$in', '$nin', '$exists', '$not', + '$regex', '$options', '$all', '$size' +]); + +function matchesValueEq(docValue, target) { + // MongoDB semantics: a scalar match also matches when the field is an + // array containing that scalar. + if (Array.isArray(docValue) && !Array.isArray(target)) { + return docValue.some(item => deepEqual(item, target)); + } + return deepEqual(docValue, target); +} + +function matchesRegex(docValue, regex) { + if (Array.isArray(docValue)) { + return docValue.some(item => typeof item === 'string' && regex.test(item)); + } + return typeof docValue === 'string' && regex.test(docValue); +} + +function matchesQuery(doc, query) { + for (const [ key, value ] of Object.entries(query)) { + if (key === '$and') { + if (!value.every(subQuery => matchesQuery(doc, subQuery))) { + return false; + } + continue; + } + if (key === '$or') { + if (!value.some(subQuery => matchesQuery(doc, subQuery))) { + return false; + } + continue; + } + if (key.startsWith('$')) { + throw new Error(`Unsupported top-level operator: ${key}`); + } + + const docValue = key === '_id' ? doc._id : getNestedField(doc, key); + + if (value instanceof RegExp) { + if (!matchesRegex(docValue, value)) { + return false; + } + continue; + } + + if (value === null || value === undefined) { + // MongoDB: { field: null } matches null, undefined, and missing + if (docValue !== null && docValue !== undefined) { + return false; + } + continue; + } + + const isOperatorObject = typeof value === 'object' && + !(value instanceof Date) && + !Array.isArray(value) && + Object.keys(value).some(k => k.startsWith('$')); + + if (!isOperatorObject) { + if (!matchesValueEq(docValue, value)) { + return false; + } + continue; + } + + for (const [ op, opValue ] of Object.entries(value)) { + if (!FIELD_OPERATORS.has(op)) { + throw new Error(`Unsupported operator: ${op}`); + } + switch (op) { + case '$eq': + // $eq null matches explicit null AND missing fields + if (opValue === null || opValue === undefined) { + if (docValue !== null && docValue !== undefined) { + return false; + } + } else if (!matchesValueEq(docValue, opValue)) { + return false; + } + break; + case '$ne': + if (opValue === null || opValue === undefined) { + if (docValue === null || docValue === undefined) { + return false; + } + } else if (matchesValueEq(docValue, opValue)) { + return false; + } + break; + case '$gt': if (!(docValue > opValue)) { + return false; + } break; + case '$gte': if (!(docValue >= opValue)) { + return false; + } break; + case '$lt': if (!(docValue < opValue)) { + return false; + } break; + case '$lte': if (!(docValue <= opValue)) { + return false; + } break; + case '$in': { + if (!Array.isArray(opValue)) { + throw new Error('$in requires an array'); + } + const matched = opValue.some(candidate => { + if (candidate instanceof RegExp) { + return matchesRegex(docValue, candidate); + } + if (candidate === null || candidate === undefined) { + return docValue === null || docValue === undefined; + } + return matchesValueEq(docValue, candidate); + }); + if (!matched) { + return false; + } + break; + } + case '$nin': { + if (!Array.isArray(opValue)) { + throw new Error('$nin requires an array'); + } + const matched = opValue.some(candidate => { + if (candidate instanceof RegExp) { + return matchesRegex(docValue, candidate); + } + if (candidate === null || candidate === undefined) { + return docValue === null || docValue === undefined; + } + return matchesValueEq(docValue, candidate); + }); + if (matched) { + return false; + } + break; + } + case '$exists': if ((docValue !== undefined) !== Boolean(opValue)) { + return false; + } break; + case '$not': { + if (typeof opValue !== 'object' || opValue === null) { + throw new Error('$not requires an object'); + } + // Apply the inner operator object to the same field and negate + // the result. Errors from unknown inner operators propagate. + if (matchesQuery(doc, { [key]: opValue })) { + return false; + } + break; + } + case '$regex': { + const pattern = opValue instanceof RegExp ? opValue.source : String(opValue); + const flags = value.$options || (opValue instanceof RegExp ? opValue.flags : ''); + const regex = new RegExp(pattern, flags); + if (!matchesRegex(docValue, regex)) { + return false; + } + break; + } + case '$options': + // Handled with $regex, skip + break; + case '$all': { + if (!Array.isArray(opValue)) { + throw new Error('$all requires an array'); + } + if (!Array.isArray(docValue)) { + return false; + } + const allPresent = opValue.every(target => + docValue.some(item => deepEqual(item, target)) + ); + if (!allPresent) { + return false; + } + break; + } + case '$size': + if (!Array.isArray(docValue) || docValue.length !== opValue) { + return false; + } + break; + } + } + } + return true; +} + +module.exports = { + AggregationCursor, + matchesQuery, + applyAggregateProject +}; diff --git a/packages/db-connect/lib/copy-database.js b/packages/db-connect/lib/copy-database.js new file mode 100644 index 0000000000..83a2df67ec --- /dev/null +++ b/packages/db-connect/lib/copy-database.js @@ -0,0 +1,11 @@ +const dump = require('./dump'); +const restore = require('./restore'); + +// Copy a database by streaming NDJSON from source to destination — the +// dump is never fully materialized in memory, so this works regardless +// of database size. +// +// Accept either URI strings or already-connected db objects (or a mix). +module.exports = async function copyDatabase(fromUriOrDb, toUriOrDb) { + await restore(toUriOrDb, dump(fromUriOrDb)); +}; diff --git a/packages/db-connect/lib/dump.js b/packages/db-connect/lib/dump.js new file mode 100644 index 0000000000..3551fa8d8e --- /dev/null +++ b/packages/db-connect/lib/dump.js @@ -0,0 +1,91 @@ +const dbConnect = require('..'); + +const BATCH_SIZE = 100; + +// Dump a database as NDJSON. Returns an AsyncIterable that yields one +// string per NDJSON record (no trailing newline). The first record of +// each collection is a header `{ _collection, _indexes? }`; subsequent +// records are `{ _collection, _doc }`. +// +// The dump is yielded incrementally so a large database can be piped to +// disk or a destination adapter without materializing the whole payload +// in memory at any point. +// +// Accept either a URI string or an already-connected db object. When a +// db object is passed, the caller owns the connection lifecycle; when a +// URI is passed, this module connects and closes the client on its own. +async function *dump(uriOrDb) { + let db; + let client; + if (typeof uriOrDb === 'string') { + client = await dbConnect(uriOrDb); + db = client.db(); + } else { + db = uriOrDb; + } + + try { + const collections = await db.listCollections().toArray(); + + for (const collInfo of collections) { + const name = collInfo.name; + const col = db.collection(name); + const indexes = await col.indexes(); + const customIndexes = indexes.filter(idx => idx.name !== '_id_'); + + const header = { _collection: name }; + if (customIndexes.length > 0) { + header._indexes = customIndexes; + } + yield JSON.stringify(header); + + let lastId = null; + while (true) { + const query = lastId ? { _id: { $gt: lastId } } : {}; + const batch = await col.find(query).sort({ _id: 1 }).limit(BATCH_SIZE).toArray(); + if (batch.length === 0) { + break; + } + for (const doc of batch) { + yield JSON.stringify({ + _collection: name, + _doc: serializeValue(doc) + }); + } + lastId = batch[batch.length - 1]._id; + if (batch.length < BATCH_SIZE) { + break; + } + } + } + } finally { + if (client) { + await client.close(); + } + } +} + +function serializeValue(obj) { + if (obj instanceof Date) { + return { $date: obj.toISOString() }; + } + if (obj === null || obj === undefined) { + return obj; + } + if (typeof obj === 'object' && obj.constructor && obj.constructor.name === 'ObjectId') { + return obj.toHexString(); + } + if (Array.isArray(obj)) { + return obj.map(serializeValue); + } + if (typeof obj === 'object') { + const result = {}; + for (const [ k, v ] of Object.entries(obj)) { + result[k] = serializeValue(v); + } + return result; + } + return obj; +} + +module.exports = dump; diff --git a/packages/apostrophe/lib/mongodb-connect.js b/packages/db-connect/lib/mongodb-connect.js similarity index 96% rename from packages/apostrophe/lib/mongodb-connect.js rename to packages/db-connect/lib/mongodb-connect.js index 1668862346..8348d77137 100644 --- a/packages/apostrophe/lib/mongodb-connect.js +++ b/packages/db-connect/lib/mongodb-connect.js @@ -29,7 +29,7 @@ module.exports = async (uri, options) => { } const records = await dns.promises.lookup('localhost', { all: true }); if (!records.length) { - // The computer that reaches this point has bigger problems 😅 + // The computer that reaches this point has bigger problems throw new Error('Unable to resolve localhost to an IP address.'); } return new Promise((resolve, reject) => { diff --git a/packages/db-connect/lib/restore.js b/packages/db-connect/lib/restore.js new file mode 100644 index 0000000000..4cf16498c9 --- /dev/null +++ b/packages/db-connect/lib/restore.js @@ -0,0 +1,166 @@ +const readline = require('readline'); +const dbConnect = require('..'); + +const BATCH_SIZE = 100; + +// Restore a database from NDJSON. `source` is streamed one line at a +// time to avoid holding the entire dump in memory. Accepted `source` +// shapes: +// +// - an AsyncIterable of NDJSON records (as produced by dump()), +// - an Iterable, +// - a Node Readable stream (e.g. process.stdin, fs.createReadStream()) — +// lines are extracted with the built-in readline module, +// - a single string containing the entire dump — retained for +// convenience in small cases; for large dumps prefer a stream. +// +// Accept either a URI string or an already-connected db object. When a +// db object is passed, the caller owns the connection lifecycle. +module.exports = async function restore(uriOrDb, source) { + let db; + let client; + if (typeof uriOrDb === 'string') { + client = await dbConnect(uriOrDb); + db = client.db(); + } else { + db = uriOrDb; + } + + try { + let col = null; + let batch = []; + + for await (const rawLine of linesOf(source)) { + const line = rawLine.trim(); + if (!line) { + continue; + } + const entry = JSON.parse(line); + + if (entry._doc) { + batch.push(deserializeValue(entry._doc)); + if (batch.length >= BATCH_SIZE) { + await col.insertMany(batch); + batch = []; + } + } else if (entry._collection) { + if (batch.length > 0) { + await col.insertMany(batch); + batch = []; + } + + col = db.collection(entry._collection); + + // Clear existing data. Use deleteMany instead of drop to + // preserve the table structure — important when the database + // is being used by a running application (e.g. Cypress tests) + try { + await col.deleteMany({}); + } catch (e) { + // Collection may not exist yet, ignore + } + + if (entry._indexes && entry._indexes.length > 0) { + for (const idx of entry._indexes) { + const options = {}; + if (idx.name) { + options.name = idx.name; + } + if (idx.unique) { + options.unique = true; + } + if (idx.sparse) { + options.sparse = true; + } + if (idx.type) { + options.type = idx.type; + } + // Text index support: preserve weights and language options + if (idx.weights) { + options.weights = idx.weights; + } + if (idx.default_language) { + options.default_language = idx.default_language; + } + if (idx.language_override) { + options.language_override = idx.language_override; + } + if (idx.textIndexVersion) { + options.textIndexVersion = idx.textIndexVersion; + } + // TTL index support + if (idx.expireAfterSeconds != null) { + options.expireAfterSeconds = idx.expireAfterSeconds; + } + await col.createIndex(idx.key, options); + } + } + } + } + + if (batch.length > 0 && col) { + await col.insertMany(batch); + } + + // For SQLite: force a WAL checkpoint to ensure all changes are + // visible to other connections immediately + if (db._sqlite) { + const result = db._sqlite.pragma('wal_checkpoint(PASSIVE)'); + if (result && result[0]) { + // Log checkpoint result for debugging + const { log, checkpointed } = result[0]; + if (log !== checkpointed) { + // Some pages weren't checkpointed, try harder + db._sqlite.pragma('wal_checkpoint(TRUNCATE)'); + } + } + } + } finally { + if (client) { + await client.close(); + } + } +}; + +// Normalize `source` into an async iterable of NDJSON lines. +function linesOf(source) { + if (source == null) { + throw new Error('restore: source is required'); + } + if (typeof source === 'string') { + return source.split('\n'); + } + // Check for Readable stream BEFORE async iterable: Node Readables are + // themselves async iterables, but iterating them yields chunks + // (Buffer/string), not lines. Use readline to split on newline. + if (typeof source.on === 'function' && typeof source.pipe === 'function') { + return readline.createInterface({ + input: source, + crlfDelay: Infinity + }); + } + if (typeof source[Symbol.asyncIterator] === 'function' || + typeof source[Symbol.iterator] === 'function') { + // Already an (async) iterable of line strings — e.g. the iterable + // returned by dump(), or an array of lines. + return source; + } + throw new Error('restore: source must be a string, iterable of lines, or a Readable stream'); +} + +function deserializeValue(obj) { + if (obj === null || typeof obj !== 'object') { + return obj; + } + if (obj.$date) { + return new Date(obj.$date); + } + if (Array.isArray(obj)) { + return obj.map(deserializeValue); + } + const result = {}; + for (const [ k, v ] of Object.entries(obj)) { + result[k] = deserializeValue(v); + } + return result; +} diff --git a/packages/db-connect/lib/shared.js b/packages/db-connect/lib/shared.js new file mode 100644 index 0000000000..6098583f6e --- /dev/null +++ b/packages/db-connect/lib/shared.js @@ -0,0 +1,438 @@ +// Shared helper functions used by both the SQLite and PostgreSQL adapters. +// These operate on plain JavaScript objects and have no database-specific logic. + +// ============================================================================= +// Value Serialization (Date handling) +// ============================================================================= + +// Recursively convert Date objects to { $date: ... } wrapper and +// undefined to null (matching MongoDB's BSON behavior where undefined +// is stored as null). This is called before JSON.stringify because +// JSON.stringify calls toJSON() on Dates before any replacer sees them, +// and omits properties with undefined values. +function serializeValue(obj) { + if (obj === undefined) { + return null; + } + if (obj === null) { + return null; + } + if (obj instanceof Date) { + return { $date: obj.toISOString() }; + } + if (Array.isArray(obj)) { + return obj.map(serializeValue); + } + if (typeof obj === 'object') { + const result = {}; + for (const [ key, value ] of Object.entries(obj)) { + result[key] = serializeValue(value); + } + return result; + } + return obj; +} + +function serializeDocument(doc) { + return JSON.stringify(serializeValue(doc)); +} + +// Convert $date wrappers back to Date objects, returning the original +// object reference when no conversions occurred in a subtree. +// Most document subtrees (rich text, widget configs) have zero dates, +// so this avoids rebuilding the entire object tree on every read. +function convertDates(obj) { + if (obj === null || typeof obj !== 'object') { + return obj; + } + if (obj.$date) { + return new Date(obj.$date); + } + if (Array.isArray(obj)) { + let changed = false; + const result = obj.map(item => { + const c = convertDates(item); + if (c !== item) { + changed = true; + } + return c; + }); + return changed ? result : obj; + } + let changed = false; + const result = {}; + for (const [ key, value ] of Object.entries(obj)) { + const c = convertDates(value); + result[key] = c; + if (c !== value) { + changed = true; + } + } + return changed ? result : obj; +} + +function deserializeDocument(data, id) { + const parsed = typeof data === 'string' ? JSON.parse(data) : data; + const doc = convertDates(parsed); + if (doc === parsed) { + // No dates found — shallow copy to add _id without mutating parsed data + return { + _id: id, + ...doc + }; + } + doc._id = id; + return doc; +} + +// ============================================================================= +// Nested Field Operations +// ============================================================================= + +function getNestedField(obj, path) { + const parts = path.split('.'); + let current = obj; + for (const part of parts) { + if (current == null) { + return undefined; + } + current = current[part]; + } + return current; +} + +function setNestedField(obj, path, value) { + const parts = path.split('.'); + if (parts.includes('__proto__')) { + return; + } + let current = obj; + for (let i = 0; i < parts.length - 1; i++) { + if (current[parts[i]] == null) { + current[parts[i]] = {}; + } + current = current[parts[i]]; + } + current[parts[parts.length - 1]] = value; +} + +function unsetNestedField(obj, path) { + const parts = path.split('.'); + let current = obj; + for (let i = 0; i < parts.length - 1; i++) { + if (current[parts[i]] == null) { + return; + } + current = current[parts[i]]; + } + delete current[parts[parts.length - 1]]; +} + +function deepEqual(a, b) { + if (a === b) { + return true; + } + if (a == null || b == null) { + return false; + } + if (typeof a !== typeof b) { + return false; + } + if (typeof a !== 'object') { + return false; + } + if (Array.isArray(a) !== Array.isArray(b)) { + return false; + } + const keysA = Object.keys(a); + const keysB = Object.keys(b); + if (keysA.length !== keysB.length) { + return false; + } + for (const key of keysA) { + if (!deepEqual(a[key], b[key])) { + return false; + } + } + return true; +} + +// ============================================================================= +// Projection (in-memory) +// ============================================================================= + +function applyProjection(doc, projection, meta = {}) { + if (!projection || Object.keys(projection).length === 0) { + return doc; + } + + // Separate $meta projections (e.g. score: { $meta: 'textScore' }) + // from regular field projections + const metaFields = Object.entries(projection).filter( + ([ k, v ]) => v && typeof v === 'object' && v.$meta + ); + const fields = Object.entries(projection).filter( + ([ k, v ]) => !(v && typeof v === 'object' && v.$meta) + ); + + const applyMeta = (result) => { + for (const [ field, spec ] of metaFields) { + if (spec.$meta === 'textScore' && meta.textScore != null) { + result[field] = meta.textScore; + } + } + return result; + }; + + if (fields.length === 0) { + return applyMeta({ ...doc }); + } + const isInclusion = fields.some(([ k, v ]) => v && k !== '_id'); + + if (isInclusion) { + const result = {}; + for (const [ field, include ] of fields) { + if (include) { + const value = getNestedField(doc, field); + if (value !== undefined) { + setNestedField(result, field, value); + } + } + } + // Always include _id unless explicitly excluded + if (projection._id !== 0 && projection._id !== false) { + result._id = doc._id; + } + return applyMeta(result); + } else { + const result = JSON.parse(JSON.stringify(doc)); + for (const [ field, include ] of fields) { + if (!include) { + unsetNestedField(result, field); + } + } + return applyMeta(result); + } +} + +// ============================================================================= +// Update Operations (in-memory) +// ============================================================================= + +function applyUpdate(doc, update) { + // Pipeline-form update: array of stages like [{ $unset: [...] }, { $set: {...} }] + if (Array.isArray(update)) { + let result = { ...doc }; + for (const stage of update) { + result = applyUpdate(result, stage); + } + return result; + } + + const result = { ...doc }; + + for (const [ op, fields ] of Object.entries(update)) { + switch (op) { + case '$set': + for (const [ field, value ] of Object.entries(fields)) { + setNestedField(result, field, value); + } + break; + case '$unset': + if (Array.isArray(fields)) { + // Pipeline form: $unset takes an array of field names + for (const field of fields) { + unsetNestedField(result, field); + } + } else { + for (const field of Object.keys(fields)) { + unsetNestedField(result, field); + } + } + break; + case '$inc': + for (const [ field, value ] of Object.entries(fields)) { + const current = getNestedField(result, field) || 0; + setNestedField(result, field, current + value); + } + break; + case '$push': + for (const [ field, value ] of Object.entries(fields)) { + const arr = getNestedField(result, field) || []; + arr.push(value); + setNestedField(result, field, arr); + } + break; + case '$pull': + for (const [ field, value ] of Object.entries(fields)) { + const arr = getNestedField(result, field) || []; + setNestedField(result, field, arr.filter(item => !deepEqual(item, value))); + } + break; + case '$addToSet': + for (const [ field, value ] of Object.entries(fields)) { + const arr = getNestedField(result, field) || []; + if (!arr.some(item => deepEqual(item, value))) { + arr.push(value); + } + setNestedField(result, field, arr); + } + break; + case '$currentDate': + for (const [ field, value ] of Object.entries(fields)) { + if (value === true || (value && value.$type === 'date')) { + setNestedField(result, field, new Date()); + } + } + break; + case '$rename': + for (const [ oldField, newField ] of Object.entries(fields)) { + const value = getNestedField(result, oldField); + if (value !== undefined) { + unsetNestedField(result, oldField); + setNestedField(result, newField, value); + } + } + break; + default: + throw new Error(`Unsupported update operator: ${op}`); + } + } + + return result; +} + +// ============================================================================= +// Anchored-Literal Prefix Extraction (for indexable regex queries) +// ============================================================================= + +// Characters that are regex metacharacters when unescaped. Anything not in +// this set is a literal and can contribute to the prefix. +const REGEX_META = new Set([ '.', '*', '+', '?', '(', ')', '[', ']', '{', '}', '|', '^', '$', '\\' ]); + +// Extract the literal prefix from a regex rooted at the start of the string, +// for use as an indexable range predicate. Returns an object +// { prefix, anchored } where `prefix` is the literal string that every match +// must begin with, and `anchored` indicates the regex begins with `^`. +// +// Returns { prefix: '', anchored: false } when no useful rewrite is possible +// (unanchored regex, case-insensitive regex, or the regex begins with a +// metacharacter). +// +// Example: +// extractAnchoredLiteralPrefix(/^\/parent\/child\/./) +// -> { prefix: '/parent/child/', anchored: true } +// extractAnchoredLiteralPrefix(/^foo.*bar/) +// -> { prefix: 'foo', anchored: true } +// extractAnchoredLiteralPrefix(/foo/) +// -> { prefix: '', anchored: false } +function extractAnchoredLiteralPrefix(regex) { + if (!(regex instanceof RegExp)) { + return { + prefix: '', + anchored: false + }; + } + // Case-insensitive regex cannot align with a case-sensitive btree range scan. + if (regex.ignoreCase) { + return { + prefix: '', + anchored: false + }; + } + const source = regex.source; + if (source.length === 0 || source[0] !== '^') { + return { + prefix: '', + anchored: false + }; + } + let i = 1; + let prefix = ''; + while (i < source.length) { + const ch = source[i]; + if (ch === '\\') { + // Escape sequence: the next character may be a literal or a character class + if (i + 1 >= source.length) { + break; + } + const next = source[i + 1]; + // Single-character escapes for a literal character (\. \/ \- \( \\ etc.) + // — anything that is a regex metacharacter, plus '/' and similar. These + // contribute the escaped character itself to the prefix. + if (REGEX_META.has(next) || next === '/' || next === '-' || next === '=' || next === '!' || next === ':' || next === ',' || next === '#' || next === ' ' || next === '"' || next === '\'' || next === '<' || next === '>' || next === '@' || next === '~' || next === '`' || next === '%' || next === '&' || next === ';') { + prefix += next; + i += 2; + continue; + } + // Any other escape (\d, \w, \s, \b, \n, \t, \uXXXX, \xXX, etc.) is not + // a simple literal — stop here. + break; + } + if (REGEX_META.has(ch)) { + // Top-level alternation (`|`) means the regex can match a completely + // different string on the other side of the pipe. Any prefix we've + // accumulated so far is not guaranteed to appear in every match, so + // we must discard it entirely. + if (ch === '|') { + prefix = ''; + break; + } + // Unescaped metacharacter ends the literal prefix. Quantifiers (`?`, + // `*`, `{`) make the *preceding* literal optional or zero-or-more, so + // that character is not guaranteed to appear in matches and must be + // dropped from the prefix. `+` guarantees at least one occurrence of + // the preceding literal, so the literal stays. + if ((ch === '?' || ch === '*' || ch === '{') && prefix.length > 0) { + prefix = prefix.slice(0, -1); + } + break; + } + prefix += ch; + i++; + } + return { + prefix, + anchored: true + }; +} + +// Given a literal prefix P, return the exclusive upper bound string U such +// that a string S starts with P iff P <= S < U. Returns null when no safe +// upper bound can be constructed (empty prefix, or last char is the maximum +// BMP code point 0xFFFF). Callers should treat a null result as "emit only +// the lower-bound predicate". +function prefixUpperBound(prefix) { + if (!prefix) { + return null; + } + const lastCode = prefix.charCodeAt(prefix.length - 1); + if (lastCode === 0xFFFF) { + return null; + } + return prefix.slice(0, -1) + String.fromCharCode(lastCode + 1); +} + +function validateInteger(value, name) { + const num = Number(value); + if (!Number.isInteger(num) || num < 0) { + throw new Error(`${name} must be a non-negative integer`); + } + return num; +} + +module.exports = { + serializeValue, + serializeDocument, + convertDates, + deserializeDocument, + getNestedField, + setNestedField, + unsetNestedField, + deepEqual, + applyProjection, + applyUpdate, + extractAnchoredLiteralPrefix, + prefixUpperBound, + validateInteger +}; diff --git a/packages/db-connect/package.json b/packages/db-connect/package.json new file mode 100644 index 0000000000..5eeedf8aac --- /dev/null +++ b/packages/db-connect/package.json @@ -0,0 +1,29 @@ +{ + "name": "@apostrophecms/db-connect", + "version": "1.0.0", + "description": "Database connection library and dump/restore tools for ApostropheCMS", + "main": "index.js", + "bin": { + "apos-db-dump": "./bin/apos-db-dump.js", + "apos-db-restore": "./bin/apos-db-restore.js" + }, + "scripts": { + "test": "mocha test/**/*.test.js --timeout 30000", + "test:mongodb": "ADAPTER=mongodb mocha test/**/*.test.js --timeout 30000", + "test:postgres": "ADAPTER=postgres mocha test/**/*.test.js --timeout 30000", + "test:sqlite": "ADAPTER=sqlite mocha test/**/*.test.js --timeout 30000", + "lint": "eslint .", + "eslint": "eslint ." + }, + "devDependencies": { + "chai": "^4.3.10", + "eslint": "^9.39.1", + "eslint-config-apostrophe": "workspace:^", + "mocha": "^11.7.1" + }, + "dependencies": { + "@apostrophecms/emulate-mongo-3-driver": "workspace:^", + "better-sqlite3": "^11.0.0", + "pg": "^8.11.3" + } +} diff --git a/packages/db-connect/test/adapter.test.js b/packages/db-connect/test/adapter.test.js new file mode 100644 index 0000000000..3d00f00122 --- /dev/null +++ b/packages/db-connect/test/adapter.test.js @@ -0,0 +1,3361 @@ +/* global describe, it, before, after, beforeEach */ +/* eslint-disable no-unused-expressions */ +const { expect } = require('chai'); + +// Test suite for the universal database adapter +// Based on actual MongoDB usage patterns in ApostropheCMS + +const ADAPTER = process.env.ADAPTER || process.env.APOS_TEST_DB_PROTOCOL || 'mongodb'; +const TEST_DB_NAME = 'dbtest-adapter'; + +describe(`Database Adapter (${ADAPTER})`, function() { + let client; + let db; + + before(async function() { + if (ADAPTER === 'mongodb') { + const mongodb = require('../adapters/mongodb'); + client = await mongodb.connect(`mongodb://localhost:27017/${TEST_DB_NAME}`); + db = client.db(); + } else if (ADAPTER === 'postgres') { + const postgres = require('../adapters/postgres'); + const user = process.env.PGUSER || process.env.USER; + const password = process.env.PGPASSWORD || ''; + const auth = password ? `${user}:${password}@` : `${user}@`; + client = await postgres.connect(`postgres://${auth}localhost:5432/dbtest_adapter`); + db = client.db(); + } else if (ADAPTER === 'multipostgres') { + const postgres = require('../adapters/postgres'); + const user = process.env.PGUSER || process.env.USER; + const password = process.env.PGPASSWORD || ''; + const auth = password ? `${user}:${password}@` : `${user}@`; + client = await postgres.connect(`multipostgres://${auth}localhost:5432/dbtest_adapter-testschema`); + db = client.db(); + } else if (ADAPTER === 'sqlite') { + const sqlite = require('../adapters/sqlite'); + const os = require('os'); + const pathModule = require('path'); + const fs = require('fs'); + const dbPath = pathModule.join(os.tmpdir(), 'dbtest-adapter.db'); + try { + fs.unlinkSync(dbPath); + } catch (e) { /* ignore */ } + client = await sqlite.connect(`sqlite://${dbPath}`); + db = client.db(); + } + }); + + after(async function() { + if (db) { + // Clean up test database + const collections = await db.listCollections().toArray(); + for (const col of collections) { + await db.collection(col.name).drop(); + } + } + if (client) { + await client.close(); + } + }); + + beforeEach(async function() { + // Clean up test collection before each test + try { + await db.collection('test').drop(); + } catch (e) { + // Collection may not exist, ignore + } + }); + + // ============================================ + // SECTION 1: Basic CRUD Operations + // ============================================ + + describe('insertOne', function() { + it('should insert a document and return insertedId', async function() { + const result = await db.collection('test').insertOne({ + _id: 'doc1', + title: 'Test Document', + value: 42 + }); + expect(result.insertedId).to.equal('doc1'); + expect(result.acknowledged).to.equal(true); + }); + + it('should auto-generate _id if not provided', async function() { + const result = await db.collection('test').insertOne({ + title: 'Auto ID Document' + }); + expect(result.insertedId).to.exist; + expect(result.acknowledged).to.equal(true); + }); + + it('should reject duplicate _id', async function() { + await db.collection('test').insertOne({ + _id: 'dup', + value: 1 + }); + try { + await db.collection('test').insertOne({ + _id: 'dup', + value: 2 + }); + expect.fail('Should have thrown duplicate key error'); + } catch (e) { + expect(e.message).to.match(/duplicate|unique|already exists/i); + } + }); + }); + + describe('insertMany', function() { + it('should insert multiple documents', async function() { + const docs = [ + { + _id: 'many1', + title: 'First' + }, + { + _id: 'many2', + title: 'Second' + }, + { + _id: 'many3', + title: 'Third' + } + ]; + const result = await db.collection('test').insertMany(docs); + expect(result.insertedCount).to.equal(3); + expect(result.acknowledged).to.equal(true); + expect(Object.keys(result.insertedIds)).to.have.lengthOf(3); + }); + + it('should reject if any document has duplicate _id', async function() { + await db.collection('test').insertOne({ + _id: 'existing', + value: 1 + }); + try { + await db.collection('test').insertMany([ + { + _id: 'new1', + value: 2 + }, + { + _id: 'existing', + value: 3 + } + ]); + expect.fail('Should have thrown duplicate key error'); + } catch (e) { + expect(e.message).to.match(/duplicate|unique|already exists/i); + } + }); + }); + + describe('findOne', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'find1', + type: 'article', + title: 'First Article', + views: 100 + }, + { + _id: 'find2', + type: 'article', + title: 'Second Article', + views: 200 + }, + { + _id: 'find3', + type: 'page', + title: 'Home Page', + views: 500 + } + ]); + }); + + it('should find a document by _id', async function() { + const doc = await db.collection('test').findOne({ _id: 'find1' }); + expect(doc).to.exist; + expect(doc._id).to.equal('find1'); + expect(doc.title).to.equal('First Article'); + }); + + it('should find a document by field value', async function() { + const doc = await db.collection('test').findOne({ type: 'page' }); + expect(doc).to.exist; + expect(doc._id).to.equal('find3'); + }); + + it('should return null if no match', async function() { + const doc = await db.collection('test').findOne({ _id: 'nonexistent' }); + expect(doc).to.be.null; + }); + + it('should support projection', async function() { + const doc = await db.collection('test').findOne( + { _id: 'find1' }, + { projection: { title: 1 } } + ); + expect(doc).to.exist; + expect(doc._id).to.equal('find1'); + expect(doc.title).to.equal('First Article'); + expect(doc.type).to.be.undefined; + expect(doc.views).to.be.undefined; + }); + + it('should support projection exclusion', async function() { + const doc = await db.collection('test').findOne( + { _id: 'find1' }, + { projection: { views: 0 } } + ); + expect(doc).to.exist; + expect(doc._id).to.equal('find1'); + expect(doc.title).to.equal('First Article'); + expect(doc.type).to.equal('article'); + expect(doc.views).to.be.undefined; + }); + }); + + describe('find', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'a1', + type: 'article', + title: 'Alpha', + order: 1, + tags: [ 'news', 'featured' ] + }, + { + _id: 'a2', + type: 'article', + title: 'Beta', + order: 2, + tags: [ 'news' ] + }, + { + _id: 'a3', + type: 'article', + title: 'Gamma', + order: 3, + tags: [ 'featured' ] + }, + { + _id: 'p1', + type: 'page', + title: 'Home', + order: 1, + tags: [] + }, + { + _id: 'p2', + type: 'page', + title: 'About', + order: 2, + tags: [ 'info' ] + } + ]); + }); + + it('should find all documents with toArray()', async function() { + const docs = await db.collection('test').find({}).toArray(); + expect(docs).to.have.lengthOf(5); + }); + + it('should find documents matching criteria', async function() { + const docs = await db.collection('test').find({ type: 'article' }).toArray(); + expect(docs).to.have.lengthOf(3); + docs.forEach(doc => expect(doc.type).to.equal('article')); + }); + + it('should support sort()', async function() { + const docs = await db.collection('test') + .find({ type: 'article' }) + .sort({ order: -1 }) + .toArray(); + expect(docs).to.have.lengthOf(3); + expect(docs[0]._id).to.equal('a3'); + expect(docs[1]._id).to.equal('a2'); + expect(docs[2]._id).to.equal('a1'); + }); + + it('should support limit()', async function() { + const docs = await db.collection('test') + .find({}) + .sort({ _id: 1 }) + .limit(2) + .toArray(); + expect(docs).to.have.lengthOf(2); + }); + + it('should support skip()', async function() { + const docs = await db.collection('test') + .find({}) + .sort({ _id: 1 }) + .skip(2) + .toArray(); + expect(docs).to.have.lengthOf(3); + }); + + it('should support skip() and limit() together', async function() { + const docs = await db.collection('test') + .find({}) + .sort({ _id: 1 }) + .skip(1) + .limit(2) + .toArray(); + expect(docs).to.have.lengthOf(2); + expect(docs[0]._id).to.equal('a2'); + expect(docs[1]._id).to.equal('a3'); + }); + + it('should support project()', async function() { + const docs = await db.collection('test') + .find({ _id: 'a1' }) + .project({ title: 1 }) + .toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('a1'); + expect(docs[0].title).to.equal('Alpha'); + expect(docs[0].type).to.be.undefined; + }); + + it('should support count()', async function() { + const count = await db.collection('test') + .find({ type: 'article' }) + .count(); + expect(count).to.equal(3); + }); + + it('should support clone()', async function() { + const cursor = db.collection('test').find({ type: 'article' }); + const cloned = cursor.clone(); + const docs1 = await cursor.toArray(); + const docs2 = await cloned.toArray(); + expect(docs1).to.have.lengthOf(3); + expect(docs2).to.have.lengthOf(3); + }); + + it('should support next() with promises', async function() { + const cursor = db.collection('test') + .find({ type: 'article' }) + .sort({ _id: 1 }); + const doc1 = await cursor.next(); + expect(doc1).to.exist; + expect(doc1._id).to.equal('a1'); + const doc2 = await cursor.next(); + expect(doc2).to.exist; + expect(doc2._id).to.equal('a2'); + const doc3 = await cursor.next(); + expect(doc3).to.exist; + expect(doc3._id).to.equal('a3'); + const doc4 = await cursor.next(); + expect(doc4).to.be.null; + }); + + it('should support next() with callbacks', function(done) { + const cursor = db.collection('test') + .find({ type: 'page' }) + .sort({ _id: 1 }); + cursor.next(function(err, doc1) { + if (err) { + return done(err); + } + expect(doc1).to.exist; + expect(doc1._id).to.equal('p1'); + cursor.next(function(err, doc2) { + if (err) { + return done(err); + } + expect(doc2).to.exist; + expect(doc2._id).to.equal('p2'); + cursor.next(function(err, doc3) { + if (err) { + return done(err); + } + expect(doc3).to.be.null; + done(); + }); + }); + }); + }); + + it('should support next() with projection', async function() { + const cursor = db.collection('test') + .find({ _id: 'a1' }) + .project({ title: 1 }); + const doc = await cursor.next(); + expect(doc).to.exist; + expect(doc._id).to.equal('a1'); + expect(doc.title).to.equal('Alpha'); + expect(doc.type).to.be.undefined; + // Close cursor to release the connection/transaction + if (cursor.close) { + await cursor.close(); + } + }); + + if (ADAPTER === 'postgres' || ADAPTER === 'multipostgres' || ADAPTER === 'sqlite') { + it('should support close() for early termination', async function() { + const cursor = db.collection('test') + .find({}) + .sort({ _id: 1 }); + const doc1 = await cursor.next(); + expect(doc1).to.exist; + await cursor.close(); + const doc2 = await cursor.next(); + expect(doc2).to.be.null; + }); + } + }); + + describe('updateOne', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'u1', + title: 'Original', + views: 10, + active: true + }, + { + _id: 'u2', + title: 'Another', + views: 20, + active: false + } + ]); + }); + + it('should update a single document', async function() { + const result = await db.collection('test').updateOne( + { _id: 'u1' }, + { $set: { title: 'Updated' } } + ); + expect(result.matchedCount).to.equal(1); + expect(result.modifiedCount).to.equal(1); + expect(result.acknowledged).to.equal(true); + + const doc = await db.collection('test').findOne({ _id: 'u1' }); + expect(doc.title).to.equal('Updated'); + expect(doc.views).to.equal(10); + }); + + it('should return matchedCount 0 if no match', async function() { + const result = await db.collection('test').updateOne( + { _id: 'nonexistent' }, + { $set: { title: 'Updated' } } + ); + expect(result.matchedCount).to.equal(0); + expect(result.modifiedCount).to.equal(0); + }); + + it('should support upsert', async function() { + const result = await db.collection('test').updateOne( + { _id: 'new1' }, + { + $set: { + title: 'Upserted', + value: 100 + } + }, + { upsert: true } + ); + expect(result.upsertedId).to.equal('new1'); + expect(result.upsertedCount).to.equal(1); + + const doc = await db.collection('test').findOne({ _id: 'new1' }); + expect(doc.title).to.equal('Upserted'); + }); + }); + + describe('updateMany', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'm1', + type: 'article', + status: 'draft' + }, + { + _id: 'm2', + type: 'article', + status: 'draft' + }, + { + _id: 'm3', + type: 'page', + status: 'draft' + } + ]); + }); + + it('should update multiple documents', async function() { + const result = await db.collection('test').updateMany( + { type: 'article' }, + { $set: { status: 'published' } } + ); + expect(result.matchedCount).to.equal(2); + expect(result.modifiedCount).to.equal(2); + + const docs = await db.collection('test').find({ status: 'published' }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + }); + + describe('replaceOne', function() { + beforeEach(async function() { + await db.collection('test').insertOne({ + _id: 'r1', + title: 'Original', + extra: 'field', + count: 5 + }); + }); + + it('should replace entire document', async function() { + const result = await db.collection('test').replaceOne( + { _id: 'r1' }, + { + _id: 'r1', + title: 'Replaced', + newField: 'value' + } + ); + expect(result.matchedCount).to.equal(1); + expect(result.modifiedCount).to.equal(1); + + const doc = await db.collection('test').findOne({ _id: 'r1' }); + expect(doc.title).to.equal('Replaced'); + expect(doc.newField).to.equal('value'); + expect(doc.extra).to.be.undefined; + expect(doc.count).to.be.undefined; + }); + + it('should support upsert', async function() { + const result = await db.collection('test').replaceOne( + { _id: 'r2' }, + { + _id: 'r2', + title: 'New Doc' + }, + { upsert: true } + ); + expect(result.upsertedId).to.equal('r2'); + }); + }); + + describe('deleteOne', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'd1', + value: 1 + }, + { + _id: 'd2', + value: 2 + } + ]); + }); + + it('should delete a single document', async function() { + const result = await db.collection('test').deleteOne({ _id: 'd1' }); + expect(result.deletedCount).to.equal(1); + expect(result.acknowledged).to.equal(true); + + const doc = await db.collection('test').findOne({ _id: 'd1' }); + expect(doc).to.be.null; + }); + + it('should return deletedCount 0 if no match', async function() { + const result = await db.collection('test').deleteOne({ _id: 'nonexistent' }); + expect(result.deletedCount).to.equal(0); + }); + }); + + describe('deleteMany', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'dm1', + type: 'temp', + value: 1 + }, + { + _id: 'dm2', + type: 'temp', + value: 2 + }, + { + _id: 'dm3', + type: 'keep', + value: 3 + } + ]); + }); + + it('should delete multiple documents', async function() { + const result = await db.collection('test').deleteMany({ type: 'temp' }); + expect(result.deletedCount).to.equal(2); + + const docs = await db.collection('test').find({}).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('dm3'); + }); + + it('should delete all documents with empty filter', async function() { + const result = await db.collection('test').deleteMany({}); + expect(result.deletedCount).to.equal(3); + }); + }); + + // ============================================ + // SECTION 2: Query Operators + // ============================================ + + describe('Query Operators', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'q1', + name: 'Alice', + age: 25, + active: true, + tags: [ 'admin', 'user' ] + }, + { + _id: 'q2', + name: 'Bob', + age: 30, + active: false, + tags: [ 'user' ] + }, + { + _id: 'q3', + name: 'Carol', + age: 35, + active: true, + tags: [ 'guest' ] + }, + { + _id: 'q4', + name: 'Dave', + age: 25, + active: true, + tags: [] + }, + { + _id: 'q5', + name: 'Eve', + age: 40, + optional: 'present', + tags: [ 'admin' ] + } + ]); + }); + + describe('Comparison Operators', function() { + it('$eq - should match equal values', async function() { + const docs = await db.collection('test').find({ age: { $eq: 25 } }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + + it('$ne - should match not equal values', async function() { + const docs = await db.collection('test').find({ age: { $ne: 25 } }).toArray(); + expect(docs).to.have.lengthOf(3); + }); + + it('$ne: null - should match docs where field exists and is not null', async function() { + // Only q5 (Eve) has the `optional` field set + const docs = await db.collection('test').find({ optional: { $ne: null } }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('q5'); + }); + + it('$eq: null - should match docs where field is null or missing', async function() { + // q1-q4 lack the `optional` field entirely + const docs = await db.collection('test').find({ optional: { $eq: null } }).toArray(); + expect(docs).to.have.lengthOf(4); + const ids = docs.map(d => d._id).sort(); + expect(ids).to.deep.equal([ 'q1', 'q2', 'q3', 'q4' ]); + }); + + it('$gt - should match greater than', async function() { + const docs = await db.collection('test').find({ age: { $gt: 30 } }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + + it('$gte - should match greater than or equal', async function() { + const docs = await db.collection('test').find({ age: { $gte: 30 } }).toArray(); + expect(docs).to.have.lengthOf(3); + }); + + it('$lt - should match less than', async function() { + const docs = await db.collection('test').find({ age: { $lt: 30 } }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + + it('$lte - should match less than or equal', async function() { + const docs = await db.collection('test').find({ age: { $lte: 30 } }).toArray(); + expect(docs).to.have.lengthOf(3); + }); + + it('$in - should match values in array', async function() { + const docs = await db.collection('test').find({ age: { $in: [ 25, 35 ] } }).toArray(); + expect(docs).to.have.lengthOf(3); + }); + + it('$nin - should match values not in array', async function() { + const docs = await db.collection('test').find({ age: { $nin: [ 25, 35 ] } }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + }); + + describe('Logical Operators', function() { + it('$and - should match all conditions', async function() { + const docs = await db.collection('test').find({ + $and: [ + { age: { $gte: 25 } }, + { active: true } + ] + }).toArray(); + expect(docs).to.have.lengthOf(3); + }); + + it('$or - should match any condition', async function() { + const docs = await db.collection('test').find({ + $or: [ + { name: 'Alice' }, + { name: 'Bob' } + ] + }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + + it('$not - should negate condition', async function() { + const docs = await db.collection('test').find({ + age: { $not: { $gt: 30 } } + }).toArray(); + expect(docs).to.have.lengthOf(3); + }); + + it('should support implicit $and with multiple fields', async function() { + const docs = await db.collection('test').find({ + age: 25, + active: true + }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + }); + + describe('Element Operators', function() { + it('$exists: true - should match documents with field', async function() { + const docs = await db.collection('test').find({ optional: { $exists: true } }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0].name).to.equal('Eve'); + }); + + it('$exists: false - should match documents without field', async function() { + const docs = await db.collection('test').find({ optional: { $exists: false } }).toArray(); + expect(docs).to.have.lengthOf(4); + }); + }); + + describe('String Operators', function() { + it('$regex - should match regex pattern', async function() { + const docs = await db.collection('test').find({ name: { $regex: /^[AB]/ } }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + + it('$regex - should support string pattern', async function() { + const docs = await db.collection('test').find({ + name: { + $regex: 'li', + $options: 'i' + } + }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0].name).to.equal('Alice'); + }); + }); + + // Anchored-literal-prefix regex queries mirror the ApostropheCMS page-tree + // descendants pattern (matchDescendants). Under the covers, the postgres + // and sqlite adapters rewrite these to an indexable range predicate + // plus a residual regex. Verify correctness here; index-usage is verified + // separately in the Indexes describe block below. + describe('Anchored regex (prefix rewrite)', function() { + beforeEach(async function() { + await db.collection('test2').deleteMany({}); + await db.collection('test2').insertMany([ + { + _id: 'p1', + path: '/' + }, + { + _id: 'p2', + path: '/parent' + }, + { + _id: 'p3', + path: '/parent/child-a' + }, + { + _id: 'p4', + path: '/parent/child-a/grandchild' + }, + { + _id: 'p5', + path: '/parent/child-b' + }, + { + _id: 'p6', + path: '/parentx' + }, // sibling, must not match a /parent/ prefix + { + _id: 'p7', + path: '/other/child' + } + ]); + }); + after(async function() { + await db.collection('test2').deleteMany({}); + }); + + it('matches exactly the descendants of a path (MongoDB semantics)', async function() { + // ApostropheCMS matchDescendants: /^\/./ + const docs = await db.collection('test2') + .find({ path: /^\/parent\/./ }) + .toArray(); + const ids = docs.map(d => d._id).sort(); + expect(ids).to.deep.equal([ 'p3', 'p4', 'p5' ]); + }); + + it('does not falsely match siblings that share a prefix without the separator', async function() { + // /parentx must not match /^\/parent\/./ even though they share '/parent' + const docs = await db.collection('test2') + .find({ path: /^\/parent\// }) + .toArray(); + const ids = docs.map(d => d._id).sort(); + expect(ids).to.deep.equal([ 'p3', 'p4', 'p5' ]); + }); + + it('excludes the parent itself from its descendants (trailing `.` enforcement)', async function() { + const docs = await db.collection('test2') + .find({ path: /^\/parent\/./ }) + .toArray(); + const ids = docs.map(d => d._id); + expect(ids).to.not.include('p2'); // /parent is not its own descendant + }); + + it('works via $regex with $options', async function() { + const docs = await db.collection('test2') + .find({ path: { $regex: '^/parent/' } }) + .toArray(); + const ids = docs.map(d => d._id).sort(); + expect(ids).to.deep.equal([ 'p3', 'p4', 'p5' ]); + }); + + it('works for _id prefix queries', async function() { + const docs = await db.collection('test2') + .find({ _id: /^p/ }) + .toArray(); + expect(docs.length).to.equal(7); + const noneMatch = await db.collection('test2') + .find({ _id: /^z/ }) + .toArray(); + expect(noneMatch).to.have.lengthOf(0); + }); + + it('returns no results when the literal prefix has no matches', async function() { + const docs = await db.collection('test2') + .find({ path: /^\/nonexistent\// }) + .toArray(); + expect(docs).to.have.lengthOf(0); + }); + + it('matches results beyond the prefix via the residual regex', async function() { + // Prefix '/parent/child-', then [ab] picks just the two direct children + const docs = await db.collection('test2') + .find({ path: /^\/parent\/child-[ab]$/ }) + .toArray(); + const ids = docs.map(d => d._id).sort(); + expect(ids).to.deep.equal([ 'p3', 'p5' ]); + }); + + it('case-insensitive anchored regex still matches correctly (no range rewrite)', async function() { + const docs = await db.collection('test2') + .find({ path: /^\/PARENT\//i }) + .toArray(); + const ids = docs.map(d => d._id).sort(); + expect(ids).to.deep.equal([ 'p3', 'p4', 'p5' ]); + }); + + it('unanchored regex still works (no rewrite)', async function() { + const docs = await db.collection('test2') + .find({ path: /child-b/ }) + .toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('p5'); + }); + }); + + describe('Array Operators', function() { + it('should match array containing value (implicit)', async function() { + const docs = await db.collection('test').find({ tags: 'admin' }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + + it('$all - should match arrays containing all values', async function() { + const docs = await db.collection('test').find({ tags: { $all: [ 'admin', 'user' ] } }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0].name).to.equal('Alice'); + }); + + it('$size - should match arrays with exact length', async function() { + const docs = await db.collection('test').find({ tags: { $size: 2 } }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0].name).to.equal('Alice'); + }); + + it('$size: 1 - should match single-element arrays', async function() { + const docs = await db.collection('test').find({ tags: { $size: 1 } }).toArray(); + expect(docs).to.have.lengthOf(3); + const names = docs.map(d => d.name).sort(); + expect(names).to.deep.equal([ 'Bob', 'Carol', 'Eve' ]); + }); + + it('$size: 0 - should match empty arrays', async function() { + const docs = await db.collection('test').find({ tags: { $size: 0 } }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0].name).to.equal('Dave'); + }); + }); + }); + + // ============================================ + // SECTION 3: Update Operators + // ============================================ + + describe('Update Operators', function() { + describe('$set', function() { + it('should set field value', async function() { + await db.collection('test').insertOne({ + _id: 'set1', + a: 1, + b: 2 + }); + await db.collection('test').updateOne({ _id: 'set1' }, { + $set: { + a: 10, + c: 3 + } + }); + const doc = await db.collection('test').findOne({ _id: 'set1' }); + expect(doc.a).to.equal(10); + expect(doc.b).to.equal(2); + expect(doc.c).to.equal(3); + }); + + it('should set nested field value', async function() { + await db.collection('test').insertOne({ + _id: 'set2', + nested: { a: 1 } + }); + await db.collection('test').updateOne({ _id: 'set2' }, { $set: { 'nested.b': 2 } }); + const doc = await db.collection('test').findOne({ _id: 'set2' }); + expect(doc.nested.a).to.equal(1); + expect(doc.nested.b).to.equal(2); + }); + }); + + describe('$unset', function() { + it('should remove field', async function() { + await db.collection('test').insertOne({ + _id: 'unset1', + a: 1, + b: 2, + c: 3 + }); + await db.collection('test').updateOne({ _id: 'unset1' }, { $unset: { b: '' } }); + const doc = await db.collection('test').findOne({ _id: 'unset1' }); + expect(doc.a).to.equal(1); + expect(doc.b).to.be.undefined; + expect(doc.c).to.equal(3); + }); + }); + + describe('$inc', function() { + it('should increment numeric field', async function() { + await db.collection('test').insertOne({ + _id: 'inc1', + count: 5 + }); + await db.collection('test').updateOne({ _id: 'inc1' }, { $inc: { count: 3 } }); + const doc = await db.collection('test').findOne({ _id: 'inc1' }); + expect(doc.count).to.equal(8); + }); + + it('should decrement with negative value', async function() { + await db.collection('test').insertOne({ + _id: 'inc2', + count: 10 + }); + await db.collection('test').updateOne({ _id: 'inc2' }, { $inc: { count: -4 } }); + const doc = await db.collection('test').findOne({ _id: 'inc2' }); + expect(doc.count).to.equal(6); + }); + + it('should create field if it does not exist', async function() { + await db.collection('test').insertOne({ _id: 'inc3' }); + await db.collection('test').updateOne({ _id: 'inc3' }, { $inc: { count: 1 } }); + const doc = await db.collection('test').findOne({ _id: 'inc3' }); + expect(doc.count).to.equal(1); + }); + }); + + describe('single-statement update path', function() { + it('should apply both $inc and $set in a single update', async function() { + await db.collection('test').insertOne({ + _id: 'combo1', + count: 5, + status: 'pending' + }); + await db.collection('test').updateOne( + { _id: 'combo1' }, + { + $inc: { count: 3 }, + $set: { status: 'done' } + } + ); + const doc = await db.collection('test').findOne({ _id: 'combo1' }); + expect(doc.count).to.equal(8); + expect(doc.status).to.equal('done'); + }); + + it('should fall back to read-modify-write for $set with upsert', async function() { + await db.collection('test').updateOne( + { _id: 'combo2' }, + { $set: { name: 'upserted' } }, + { upsert: true } + ); + const doc = await db.collection('test').findOne({ _id: 'combo2' }); + expect(doc.name).to.equal('upserted'); + }); + + it('should fall back to read-modify-write when $set touches text-indexed fields', async function() { + const col = db.collection('test_text_atomic'); + await col.insertOne({ + _id: 'ta1', + title: 'Original', + highSearchText: 'original', + lowSearchText: 'original', + searchBoost: '', + body: 'test' + }); + await col.createIndex({ + highSearchText: 'text', + lowSearchText: 'text', + title: 'text', + searchBoost: 'text' + }); + // This $set touches 'title' which is text-indexed, so it must + // go through the read-modify-write path to keep FTS in sync + await col.updateOne( + { _id: 'ta1' }, + { + $set: { + title: 'Updated', + highSearchText: 'updated' + } + } + ); + const doc = await col.findOne({ _id: 'ta1' }); + expect(doc.title).to.equal('Updated'); + // Verify text search finds the updated content + const found = await col.find({ $text: { $search: 'updated' } }).toArray(); + expect(found).to.have.lengthOf(1); + expect(found[0]._id).to.equal('ta1'); + // lowSearchText still contains 'original' so it should still match + // (FTS indexes all text fields, not just the ones we updated) + const stillFound = await col.find({ $text: { $search: 'original' } }).toArray(); + expect(stillFound).to.have.lengthOf(1); + }); + + it('should use single-statement path when $set does not touch text-indexed fields', async function() { + const col = db.collection('test_text_atomic'); + // 'body' is not text-indexed, so this should use the single-statement path + await col.updateOne( + { _id: 'ta1' }, + { + $set: { body: 'changed' }, + $inc: { views: 1 } + } + ); + const doc = await col.findOne({ _id: 'ta1' }); + expect(doc.body).to.equal('changed'); + expect(doc.views).to.equal(1); + // Text search should still work (FTS not affected) + const found = await col.find({ $text: { $search: 'updated' } }).toArray(); + expect(found).to.have.lengthOf(1); + }); + + it('should handle $unset via single-statement path', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_unset1', + a: 1, + b: 2, + c: 3 + }); + await db.collection('test').updateOne( + { _id: 'atomic_unset1' }, + { $unset: { b: '' } } + ); + const doc = await db.collection('test').findOne({ _id: 'atomic_unset1' }); + expect(doc.a).to.equal(1); + expect(doc.b).to.be.undefined; + expect(doc.c).to.equal(3); + }); + + it('should handle $unset combined with $set and $inc in a single update', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_unset2', + keep: 'yes', + remove: 'gone', + count: 0 + }); + await db.collection('test').updateOne( + { _id: 'atomic_unset2' }, + { + $set: { keep: 'updated' }, + $unset: { remove: '' }, + $inc: { count: 1 } + } + ); + const doc = await db.collection('test').findOne({ _id: 'atomic_unset2' }); + expect(doc.keep).to.equal('updated'); + expect(doc.remove).to.be.undefined; + expect(doc.count).to.equal(1); + }); + + it('should handle $currentDate combined with $set in a single update', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_cd1', + status: 'pending' + }); + const before = new Date(); + await db.collection('test').updateOne( + { _id: 'atomic_cd1' }, + { + $set: { status: 'done' }, + $currentDate: { updatedAt: true } + } + ); + const after = new Date(); + const doc = await db.collection('test').findOne({ _id: 'atomic_cd1' }); + expect(doc.status).to.equal('done'); + expect(doc.updatedAt).to.be.an.instanceOf(Date); + expect(doc.updatedAt.getTime()).to.be.at.least(before.getTime()); + expect(doc.updatedAt.getTime()).to.be.at.most(after.getTime()); + }); + + it('should handle scalar $addToSet via single-statement path', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_addset1', + tags: [ 'a', 'b' ] + }); + // Add new value + await db.collection('test').updateOne( + { _id: 'atomic_addset1' }, + { $addToSet: { tags: 'c' } } + ); + let doc = await db.collection('test').findOne({ _id: 'atomic_addset1' }); + expect(doc.tags).to.include.members([ 'a', 'b', 'c' ]); + expect(doc.tags).to.have.lengthOf(3); + // Add duplicate (should be no-op) + await db.collection('test').updateOne( + { _id: 'atomic_addset1' }, + { $addToSet: { tags: 'b' } } + ); + doc = await db.collection('test').findOne({ _id: 'atomic_addset1' }); + expect(doc.tags).to.have.lengthOf(3); + }); + + it('should handle scalar $pull via single-statement path', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_pull1', + tags: [ 'a', 'b', 'c' ] + }); + await db.collection('test').updateOne( + { _id: 'atomic_pull1' }, + { $pull: { tags: 'b' } } + ); + const doc = await db.collection('test').findOne({ _id: 'atomic_pull1' }); + expect(doc.tags).to.deep.equal([ 'a', 'c' ]); + }); + + it('should handle scalar $push via single-statement path', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_push1', + tags: [ 'a', 'b' ] + }); + await db.collection('test').updateOne( + { _id: 'atomic_push1' }, + { $push: { tags: 'c' } } + ); + const doc = await db.collection('test').findOne({ _id: 'atomic_push1' }); + expect(doc.tags).to.deep.equal([ 'a', 'b', 'c' ]); + }); + + it('should handle $push when field does not exist', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_push2', + name: 'test' + }); + await db.collection('test').updateOne( + { _id: 'atomic_push2' }, + { $push: { tags: 'first' } } + ); + const doc = await db.collection('test').findOne({ _id: 'atomic_push2' }); + expect(doc.tags).to.deep.equal([ 'first' ]); + }); + + it('should handle $push with $set and $inc combined', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_push3', + log: [ 'init' ], + count: 0, + status: 'new' + }); + await db.collection('test').updateOne( + { _id: 'atomic_push3' }, + { + $push: { log: 'step1' }, + $inc: { count: 1 }, + $set: { status: 'running' } + } + ); + const doc = await db.collection('test').findOne({ _id: 'atomic_push3' }); + expect(doc.log).to.deep.equal([ 'init', 'step1' ]); + expect(doc.count).to.equal(1); + expect(doc.status).to.equal('running'); + }); + + it('should handle $pull and $addToSet on different fields', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_both1', + active: [ 'x', 'y' ], + archived: [ 'z' ] + }); + await db.collection('test').updateOne( + { _id: 'atomic_both1' }, + { + $pull: { active: 'x' }, + $addToSet: { archived: 'x' } + } + ); + const doc = await db.collection('test').findOne({ _id: 'atomic_both1' }); + expect(doc.active).to.deep.equal([ 'y' ]); + expect(doc.archived).to.include.members([ 'z', 'x' ]); + }); + + it('should handle $addToSet when field does not exist', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_addset2', + name: 'test' + }); + await db.collection('test').updateOne( + { _id: 'atomic_addset2' }, + { $addToSet: { tags: 'first' } } + ); + const doc = await db.collection('test').findOne({ _id: 'atomic_addset2' }); + expect(doc.tags).to.deep.equal([ 'first' ]); + }); + + it('should handle $pull when field does not exist', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_pull2', + name: 'test' + }); + await db.collection('test').updateOne( + { _id: 'atomic_pull2' }, + { $pull: { tags: 'nope' } } + ); + const doc = await db.collection('test').findOne({ _id: 'atomic_pull2' }); + // MongoDB leaves the field absent, SQL adapters create an + // empty array. Both are acceptable — the field has no elements. + if (doc.tags !== undefined) { + expect(doc.tags).to.deep.equal([]); + } + }); + + it('should combine scalar $pull with $set and $inc', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_combo3', + ids: [ 'a', 'b' ], + count: 5, + status: 'active' + }); + await db.collection('test').updateOne( + { _id: 'atomic_combo3' }, + { + $pull: { ids: 'a' }, + $inc: { count: -1 }, + $set: { status: 'modified' } + } + ); + const doc = await db.collection('test').findOne({ _id: 'atomic_combo3' }); + expect(doc.ids).to.deep.equal([ 'b' ]); + expect(doc.count).to.equal(4); + expect(doc.status).to.equal('modified'); + }); + + it('should fall back to read-modify-write for object $addToSet', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_objset1', + items: [ { id: 1 } ] + }); + await db.collection('test').updateOne( + { _id: 'atomic_objset1' }, + { $addToSet: { items: { id: 2 } } } + ); + const doc = await db.collection('test').findOne({ _id: 'atomic_objset1' }); + expect(doc.items).to.have.lengthOf(2); + expect(doc.items[1].id).to.equal(2); + }); + + it('should fall back to read-modify-write for object $pull', async function() { + await db.collection('test').insertOne({ + _id: 'atomic_objpull1', + items: [ + { id: 1 }, + { id: 2 } + ] + }); + await db.collection('test').updateOne( + { _id: 'atomic_objpull1' }, + { $pull: { items: { id: 1 } } } + ); + const doc = await db.collection('test').findOne({ _id: 'atomic_objpull1' }); + expect(doc.items).to.have.lengthOf(1); + expect(doc.items[0].id).to.equal(2); + }); + }); + + describe('$push', function() { + it('should add element to array', async function() { + await db.collection('test').insertOne({ + _id: 'push1', + items: [ 'a', 'b' ] + }); + await db.collection('test').updateOne({ _id: 'push1' }, { $push: { items: 'c' } }); + const doc = await db.collection('test').findOne({ _id: 'push1' }); + expect(doc.items).to.deep.equal([ 'a', 'b', 'c' ]); + }); + + it('should create array if it does not exist', async function() { + await db.collection('test').insertOne({ _id: 'push2' }); + await db.collection('test').updateOne({ _id: 'push2' }, { $push: { items: 'a' } }); + const doc = await db.collection('test').findOne({ _id: 'push2' }); + expect(doc.items).to.deep.equal([ 'a' ]); + }); + }); + + describe('$pull', function() { + it('should remove matching elements from array', async function() { + await db.collection('test').insertOne({ + _id: 'pull1', + items: [ 'a', 'b', 'c', 'b' ] + }); + await db.collection('test').updateOne({ _id: 'pull1' }, { $pull: { items: 'b' } }); + const doc = await db.collection('test').findOne({ _id: 'pull1' }); + expect(doc.items).to.deep.equal([ 'a', 'c' ]); + }); + }); + + describe('$addToSet', function() { + it('should add element only if not present', async function() { + await db.collection('test').insertOne({ + _id: 'add1', + tags: [ 'a', 'b' ] + }); + await db.collection('test').updateOne({ _id: 'add1' }, { $addToSet: { tags: 'c' } }); + const doc = await db.collection('test').findOne({ _id: 'add1' }); + expect(doc.tags).to.deep.equal([ 'a', 'b', 'c' ]); + }); + + it('should not add duplicate element', async function() { + await db.collection('test').insertOne({ + _id: 'add2', + tags: [ 'a', 'b' ] + }); + await db.collection('test').updateOne({ _id: 'add2' }, { $addToSet: { tags: 'b' } }); + const doc = await db.collection('test').findOne({ _id: 'add2' }); + expect(doc.tags).to.deep.equal([ 'a', 'b' ]); + }); + }); + + describe('$currentDate', function() { + it('should set field to current date', async function() { + await db.collection('test').insertOne({ + _id: 'date1', + name: 'test' + }); + const before = new Date(); + await db.collection('test').updateOne({ _id: 'date1' }, { $currentDate: { updatedAt: true } }); + const after = new Date(); + const doc = await db.collection('test').findOne({ _id: 'date1' }); + expect(doc.updatedAt).to.be.instanceOf(Date); + expect(doc.updatedAt.getTime()).to.be.at.least(before.getTime()); + expect(doc.updatedAt.getTime()).to.be.at.most(after.getTime()); + }); + }); + }); + + // ============================================ + // SECTION 4: Counting and Distinct + // ============================================ + + describe('countDocuments', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'c1', + type: 'a', + value: 1 + }, + { + _id: 'c2', + type: 'a', + value: 2 + }, + { + _id: 'c3', + type: 'b', + value: 3 + } + ]); + }); + + it('should count all documents', async function() { + const count = await db.collection('test').countDocuments({}); + expect(count).to.equal(3); + }); + + it('should count matching documents', async function() { + const count = await db.collection('test').countDocuments({ type: 'a' }); + expect(count).to.equal(2); + }); + }); + + describe('distinct', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'd1', + category: 'food', + tag: 'healthy' + }, + { + _id: 'd2', + category: 'food', + tag: 'junk' + }, + { + _id: 'd3', + category: 'tech', + tag: 'healthy' + }, + { + _id: 'd4', + category: 'tech', + tag: 'new' + } + ]); + }); + + it('should return distinct values for field', async function() { + const values = await db.collection('test').distinct('category'); + expect(values.sort()).to.deep.equal([ 'food', 'tech' ]); + }); + + it('should return distinct values with filter', async function() { + const values = await db.collection('test').distinct('tag', { category: 'food' }); + expect(values.sort()).to.deep.equal([ 'healthy', 'junk' ]); + }); + + it('should return distinct object values as parsed objects', async function() { + await db.collection('test').insertMany([ + { + _id: 'dobj1', + updatedBy: { _id: 'user1', title: 'Alice' } + }, + { + _id: 'dobj2', + updatedBy: { _id: 'user2', title: 'Bob' } + }, + { + _id: 'dobj3', + updatedBy: { _id: 'user1', title: 'Alice' } + } + ]); + const values = await db.collection('test').distinct('updatedBy'); + // Should return parsed objects, not JSON strings + expect(values).to.be.an('array'); + expect(values.length).to.be.at.least(2); + const hasUser1 = values.some(v => typeof v === 'object' && v._id === 'user1'); + const hasUser2 = values.some(v => typeof v === 'object' && v._id === 'user2'); + expect(hasUser1).to.equal(true); + expect(hasUser2).to.equal(true); + }); + }); + + // ============================================ + // SECTION 5: Aggregation (Limited) + // ============================================ + + describe('aggregate', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'agg1', + category: 'fruit', + name: 'apple', + qty: 10 + }, + { + _id: 'agg2', + category: 'fruit', + name: 'banana', + qty: 5 + }, + { + _id: 'agg3', + category: 'vegetable', + name: 'carrot', + qty: 8 + }, + { + _id: 'agg4', + category: 'vegetable', + name: 'broccoli', + qty: 3 + } + ]); + }); + + it('$match - should filter documents', async function() { + const results = await db.collection('test').aggregate([ + { $match: { category: 'fruit' } } + ]).toArray(); + expect(results).to.have.lengthOf(2); + }); + + it('$group - should group and aggregate', async function() { + const results = await db.collection('test').aggregate([ + { + $group: { + _id: '$category', + total: { $sum: '$qty' } + } + } + ]).toArray(); + expect(results).to.have.lengthOf(2); + const fruit = results.find(r => r._id === 'fruit'); + const vegetable = results.find(r => r._id === 'vegetable'); + expect(fruit.total).to.equal(15); + expect(vegetable.total).to.equal(11); + }); + + it('$project - should project fields', async function() { + const results = await db.collection('test').aggregate([ + { $match: { _id: 'agg1' } }, + { + $project: { + name: 1, + qty: 1 + } + } + ]).toArray(); + expect(results).to.have.lengthOf(1); + expect(results[0].name).to.equal('apple'); + expect(results[0].category).to.be.undefined; + }); + + it('$unwind - should unwind array field', async function() { + await db.collection('test').insertOne({ + _id: 'agg5', + name: 'mixed', + items: [ 'x', 'y', 'z' ] + }); + const results = await db.collection('test').aggregate([ + { $match: { _id: 'agg5' } }, + { $unwind: '$items' } + ]).toArray(); + expect(results).to.have.lengthOf(3); + expect(results.map(r => r.items)).to.deep.equal([ 'x', 'y', 'z' ]); + }); + + it('$match - throws on unrecognized operator (parity with find())', async function() { + // The in-memory matcher backs any $match stage after the first, + // so unknown operators must throw instead of silently matching + // everything — matching the SQL find() path. + let err; + try { + await db.collection('test').aggregate([ + { $match: { category: 'fruit' } }, + { $match: { qty: { $madeUp: 5 } } } + ]).toArray(); + } catch (e) { + err = e; + } + expect(err).to.exist; + expect(err.message).to.match(/\$madeUp|Unsupported operator/); + }); + + it('$match - supports $regex/$not/$all/$size in the in-memory matcher', async function() { + await db.collection('test').insertMany([ + { + _id: 'agg6', + name: 'match', + tags: [ 'a', 'b', 'c' ] + }, + { + _id: 'agg7', + name: 'other', + tags: [ 'a' ] + } + ]); + const r1 = await db.collection('test').aggregate([ + { $match: { name: { $exists: true } } }, + { $match: { name: { $regex: '^mat' } } } + ]).toArray(); + expect(r1.map(d => d._id)).to.deep.equal([ 'agg6' ]); + + const r2 = await db.collection('test').aggregate([ + { $match: { _id: { $in: [ 'agg6', 'agg7' ] } } }, + { $match: { tags: { $size: 3 } } } + ]).toArray(); + expect(r2.map(d => d._id)).to.deep.equal([ 'agg6' ]); + + const r3 = await db.collection('test').aggregate([ + { $match: { _id: { $in: [ 'agg6', 'agg7' ] } } }, + { $match: { tags: { $all: [ 'a', 'b' ] } } } + ]).toArray(); + expect(r3.map(d => d._id)).to.deep.equal([ 'agg6' ]); + + const r4 = await db.collection('test').aggregate([ + { $match: { _id: { $in: [ 'agg6', 'agg7' ] } } }, + { $match: { name: { $not: { $regex: '^mat' } } } } + ]).toArray(); + expect(r4.map(d => d._id)).to.deep.equal([ 'agg7' ]); + }); + }); + + // ============================================ + // SECTION 6: Index Operations + // ============================================ + + describe('Index Operations', function() { + it('createIndex - should create a single field index', async function() { + await db.collection('test').insertOne({ + _id: 'idx1', + field: 'value' + }); + const indexName = await db.collection('test').createIndex({ field: 1 }); + expect(indexName).to.be.a('string'); + + const indexes = await db.collection('test').indexes(); + const fieldIndex = indexes.find(i => i.key && i.key.field === 1); + expect(fieldIndex).to.exist; + }); + + it('createIndex - should create a compound index', async function() { + await db.collection('test').insertOne({ + _id: 'idx2', + a: 1, + b: 2 + }); + const indexName = await db.collection('test').createIndex({ + a: 1, + b: -1 + }); + expect(indexName).to.be.a('string'); + }); + + it('createIndex - should create a unique index', async function() { + await db.collection('test').insertOne({ + _id: 'idx3', + email: 'test@example.com' + }); + await db.collection('test').createIndex({ email: 1 }, { unique: true }); + + // Should reject duplicate + try { + await db.collection('test').insertOne({ + _id: 'idx4', + email: 'test@example.com' + }); + expect.fail('Should have thrown duplicate key error'); + } catch (e) { + expect(e.message).to.match(/duplicate|unique|already exists/i); + } + }); + + it('createIndex - should support text index', async function() { + await db.collection('test').insertOne({ + _id: 'txt1', + content: 'hello world' + }); + const indexName = await db.collection('test').createIndex({ content: 'text' }); + expect(indexName).to.be.a('string'); + }); + + it('dropIndex - should drop an index', async function() { + await db.collection('test').insertOne({ + _id: 'drop1', + field: 'value' + }); + const indexName = await db.collection('test').createIndex({ field: 1 }); + + await db.collection('test').dropIndex(indexName); + + const indexes = await db.collection('test').indexes(); + const fieldIndex = indexes.find(i => i.name === indexName); + expect(fieldIndex).to.not.exist; + }); + + it('indexes - should list all indexes', async function() { + await db.collection('test').insertOne({ + _id: 'list1', + a: 1, + b: 2 + }); + await db.collection('test').createIndex({ a: 1 }); + await db.collection('test').createIndex({ b: 1 }); + + const indexes = await db.collection('test').indexes(); + expect(indexes.length).to.be.at.least(3); // _id index + a + b + }); + + it('createIndex - should create index on nested field', async function() { + await db.collection('test').insertOne({ + _id: 'nested1', + user: { profile: { name: 'Alice' } } + }); + const indexName = await db.collection('test').createIndex({ 'user.profile.name': 1 }); + expect(indexName).to.be.a('string'); + + // Verify we can query using the indexed field + const doc = await db.collection('test').findOne({ 'user.profile.name': 'Alice' }); + expect(doc).to.exist; + expect(doc._id).to.equal('nested1'); + }); + + it('createIndex - should create sparse index', async function() { + // Insert docs with and without the indexed field + await db.collection('test').insertMany([ + { + _id: 'sparse1', + optionalField: 'present' + }, + { _id: 'sparse2' }, // no optionalField + { + _id: 'sparse3', + optionalField: 'also present' + } + ]); + + const indexName = await db.collection('test').createIndex( + { optionalField: 1 }, + { sparse: true } + ); + expect(indexName).to.be.a('string'); + + // Both docs with the field should be findable + const docs = await db.collection('test').find({ optionalField: { $exists: true } }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + + it('createIndex - should create unique sparse index', async function() { + // Unique sparse index allows multiple docs without the field + await db.collection('test').insertMany([ + { + _id: 'us1', + uniqueOptional: 'value1' + }, + { _id: 'us2' }, // no uniqueOptional - allowed with sparse + { _id: 'us3' } // no uniqueOptional - also allowed with sparse + ]); + + await db.collection('test').createIndex( + { uniqueOptional: 1 }, + { + unique: true, + sparse: true + } + ); + + // Should reject duplicate value + try { + await db.collection('test').insertOne({ + _id: 'us4', + uniqueOptional: 'value1' + }); + expect.fail('Should have thrown duplicate key error'); + } catch (e) { + expect(e.message).to.match(/duplicate|unique|already exists/i); + } + + // But allow another doc without the field + await db.collection('test').insertOne({ _id: 'us5' }); + const count = await db.collection('test').countDocuments({ _id: { $in: [ 'us2', 'us3', 'us5' ] } }); + expect(count).to.equal(3); + }); + + // Typed index tests - these are PostgreSQL-specific optimizations + // but should work (be ignored) for MongoDB as well + it('createIndex - should create numeric index for range queries', async function() { + await db.collection('test').insertMany([ + { + _id: 'num1', + price: 10 + }, + { + _id: 'num2', + price: 25 + }, + { + _id: 'num3', + price: 50 + }, + { + _id: 'num4', + price: 100 + } + ]); + + // Create numeric index for efficient range queries + const indexName = await db.collection('test').createIndex( + { price: 1 }, + { type: 'number' } + ); + expect(indexName).to.be.a('string'); + + // Range queries should work correctly + const cheap = await db.collection('test').find({ price: { $lt: 30 } }).toArray(); + expect(cheap).to.have.lengthOf(2); + expect(cheap.map(d => d._id).sort()).to.deep.equal([ 'num1', 'num2' ]); + + const expensive = await db.collection('test').find({ price: { $gte: 50 } }).toArray(); + expect(expensive).to.have.lengthOf(2); + expect(expensive.map(d => d._id).sort()).to.deep.equal([ 'num3', 'num4' ]); + + const midRange = await db.collection('test').find({ + price: { + $gt: 10, + $lt: 100 + } + }).toArray(); + expect(midRange).to.have.lengthOf(2); + expect(midRange.map(d => d._id).sort()).to.deep.equal([ 'num2', 'num3' ]); + }); + + it('createIndex - should create date index for range queries', async function() { + const now = new Date(); + const yesterday = new Date(now.getTime() - 24 * 60 * 60 * 1000); + const lastWeek = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000); + const lastMonth = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000); + + await db.collection('test').insertMany([ + { + _id: 'date1', + createdAt: lastMonth + }, + { + _id: 'date2', + createdAt: lastWeek + }, + { + _id: 'date3', + createdAt: yesterday + }, + { + _id: 'date4', + createdAt: now + } + ]); + + // Create date index for efficient range queries + const indexName = await db.collection('test').createIndex( + { createdAt: 1 }, + { type: 'date' } + ); + expect(indexName).to.be.a('string'); + + // Range queries should work correctly + const recent = await db.collection('test').find({ + createdAt: { $gte: yesterday } + }).toArray(); + expect(recent).to.have.lengthOf(2); + expect(recent.map(d => d._id).sort()).to.deep.equal([ 'date3', 'date4' ]); + + const older = await db.collection('test').find({ + createdAt: { $lt: lastWeek } + }).toArray(); + expect(older).to.have.lengthOf(1); + expect(older[0]._id).to.equal('date1'); + + const midRange = await db.collection('test').find({ + createdAt: { + $gt: lastMonth, + $lt: now + } + }).toArray(); + expect(midRange).to.have.lengthOf(2); + expect(midRange.map(d => d._id).sort()).to.deep.equal([ 'date2', 'date3' ]); + }); + + it('createIndex - should create unique numeric index', async function() { + await db.collection('test').insertMany([ + { + _id: 'unum1', + rank: 1 + }, + { + _id: 'unum2', + rank: 2 + } + ]); + + await db.collection('test').createIndex( + { rank: 1 }, + { + type: 'number', + unique: true + } + ); + + // Should reject duplicate + try { + await db.collection('test').insertOne({ + _id: 'unum3', + rank: 1 + }); + expect.fail('Should have thrown duplicate key error'); + } catch (e) { + expect(e.message).to.match(/duplicate|unique|already exists/i); + } + + // Different value should work + await db.collection('test').insertOne({ + _id: 'unum3', + rank: 3 + }); + const count = await db.collection('test').countDocuments({ _id: { $regex: '^unum' } }); + expect(count).to.equal(3); + }); + + if (ADAPTER === 'postgres' || ADAPTER === 'multipostgres') { + // PostgreSQL-specific: verify indexes() reconstructs metadata from + // pg_indexes when the in-memory _indexes Map is not populated + // (e.g. after a reconnect) + it('indexes - should reconstruct index metadata from SQL definitions', async function() { + await db.collection('test').insertOne({ + _id: 'parse1', + slug: 'hello', + price: 42, + createdAt: new Date('2024-06-01T00:00:00Z'), + user: { profile: { name: 'Alice' } }, + content: 'some text here' + }); + + // Create various index types + await db.collection('test').createIndex({ slug: 1 }); + await db.collection('test').createIndex({ + a: 1, + b: -1 + }); + await db.collection('test').createIndex({ price: 1 }, { type: 'number' }); + await db.collection('test').createIndex({ createdAt: 1 }, { type: 'date' }); + await db.collection('test').createIndex({ 'user.profile.name': 1 }); + await db.collection('test').createIndex({ content: 'text' }); + await db.collection('test').createIndex({ slug: 1 }, { + unique: true, + name: 'slug_unique' + }); + await db.collection('test').createIndex({ price: 1 }, { + sparse: true, + type: 'number', + name: 'price_sparse' + }); + + // Clear the in-memory index cache to force parsing from SQL + db.collection('test')._indexes.clear(); + + const indexes = await db.collection('test').indexes(); + + // Find the default text index + const textIdx = indexes.find(i => i.key && i.key.content === 'text'); + expect(textIdx).to.exist; + expect(textIdx.key.content).to.equal('text'); + + // Find the single field index + const slugIdx = indexes.find(i => + i.key && i.key.slug === 1 && !i.unique + ); + expect(slugIdx).to.exist; + + // Find the numeric typed index + const numIdx = indexes.find(i => + i.key && i.key.price === 1 && i.type === 'number' && !i.sparse + ); + expect(numIdx).to.exist; + + // Find the date typed index + const dateIdx = indexes.find(i => + i.key && i.key.createdAt === 1 && i.type === 'date' + ); + expect(dateIdx).to.exist; + + // Find the nested field index + const nestedIdx = indexes.find(i => + i.key && i.key['user.profile.name'] === 1 + ); + expect(nestedIdx).to.exist; + + // Find the compound index + const compoundIdx = indexes.find(i => + i.key && i.key.a === 1 && i.key.b === -1 + ); + expect(compoundIdx).to.exist; + + // Find the unique index + const uniqueIdx = indexes.find(i => + i.key && i.key.slug === 1 && i.unique + ); + expect(uniqueIdx).to.exist; + + // Find the sparse numeric index + const sparseNumIdx = indexes.find(i => + i.key && i.key.price === 1 && i.sparse && i.type === 'number' + ); + expect(sparseNumIdx).to.exist; + }); + } + + // Verify that anchored-regex queries (e.g. ApostropheCMS matchDescendants) + // actually use a btree index on the matched field via the rewrite to a + // range predicate. This is the behavior we care about — without it, page + // tree queries degrade to O(n) sequential scans. + // Seed a collection with many non-matching _ids plus two targets + // whose _id matches /^\/tree\/parent\//. We assert index usage + // against the primary-key _id column rather than a JSONB field, + // because JSON-field regex queries carry a scalar-OR-array-element + // disjunction (MongoDB semantics for regex-matches-array-of-strings) + // that deliberately blocks bitmap index scans on the scalar branch. + // The primary-key path has no such fallback, so it is the right + // place to prove that the anchored-regex → range-predicate rewrite + // makes the adapter's real query index-eligible. + async function seedIdIndexCollection(coll) { + await coll.deleteMany({}); + const docs = []; + for (let i = 0; i < 200; i++) { + docs.push({ _id: `/p${i}/child` }); + } + docs.push({ _id: '/tree/parent/a' }); + docs.push({ _id: '/tree/parent/b' }); + await coll.insertMany(docs); + } + + if (ADAPTER === 'postgres' || ADAPTER === 'multipostgres') { + it('anchored regex on an indexed field uses a btree index scan (postgres)', async function() { + const coll = db.collection('pathidx'); + await seedIdIndexCollection(coll); + + // End-to-end correctness first — proves the rewrite produces + // correct results. + const results = await coll.find({ _id: /^\/tree\/parent\// }).toArray(); + const ids = results.map(d => d._id).sort(); + expect(ids).to.deep.equal([ '/tree/parent/a', '/tree/parent/b' ]); + + // Ask the cursor for the SQL it would actually run, then EXPLAIN + // THAT SQL. This keeps the test honest: future changes to + // buildWhereClause immediately show up here instead of drifting + // apart from a hand-written SQL string. + const cursor = coll.find({ _id: /^\/tree\/parent\// }); + const { sql, params } = await cursor.explain(); + + const pool = db._pool; + const client = await pool.connect(); + try { + await client.query('BEGIN'); + // Make sure the planner has stats for the freshly seeded + // table — otherwise it may default to a seqscan based on a + // 0-row estimate. + await client.query(`ANALYZE ${coll._qualifiedName()}`); + // Force enable_seqscan = off so the planner must use the + // index if it is eligible at all. If it still falls back to + // a sequential scan under this setting, the rewrite is + // producing a predicate the planner cannot match to the + // index — i.e. the regression we are guarding against. + await client.query('SET LOCAL enable_seqscan = off'); + const explain = await client.query(`EXPLAIN ${sql}`, params); + const planText = explain.rows.map(r => r['QUERY PLAN']).join('\n'); + expect(planText).to.match(/Index (Only )?Scan|Bitmap Index Scan/); + await client.query('ROLLBACK'); + } finally { + client.release(); + } + + await coll.deleteMany({}); + }); + } + + if (ADAPTER === 'sqlite') { + it('anchored regex on an indexed field uses a btree index search (sqlite)', async function() { + const coll = db.collection('pathidx'); + await seedIdIndexCollection(coll); + + // End-to-end correctness first + const results = await coll.find({ _id: /^\/tree\/parent\// }).toArray(); + const ids = results.map(d => d._id).sort(); + expect(ids).to.deep.equal([ '/tree/parent/a', '/tree/parent/b' ]); + + // Ask the cursor for the SQL it would actually run, then + // EXPLAIN QUERY PLAN on that same SQL. SQLite plan output: + // "SEARCH table USING INDEX idx_name (...)" -- uses index + // "SCAN table" -- full scan + const cursor = coll.find({ _id: /^\/tree\/parent\// }); + const { sql, params } = await cursor.explain(); + const sqlite = db._sqlite; + const planRows = sqlite.prepare(`EXPLAIN QUERY PLAN ${sql}`).all(...params); + const planText = planRows.map(r => r.detail || '').join('\n'); + expect(planText).to.match(/SEARCH.*USING (INDEX|ROWID|PRIMARY KEY)/); + + await coll.deleteMany({}); + }); + } + }); + + // ============================================ + // SECTION 7: Bulk Operations + // ============================================ + + describe('bulkWrite', function() { + it('should execute multiple operations', async function() { + await db.collection('test').insertOne({ + _id: 'bulk1', + value: 1 + }); + + const result = await db.collection('test').bulkWrite([ + { + insertOne: { + document: { + _id: 'bulk2', + value: 2 + } + } + }, + { + updateOne: { + filter: { _id: 'bulk1' }, + update: { $set: { value: 10 } } + } + }, + { + insertOne: { + document: { + _id: 'bulk3', + value: 3 + } + } + }, + { deleteOne: { filter: { _id: 'bulk3' } } } + ]); + + expect(result.insertedCount).to.equal(2); + expect(result.modifiedCount).to.equal(1); + expect(result.deletedCount).to.equal(1); + + const docs = await db.collection('test').find({}).toArray(); + expect(docs).to.have.lengthOf(2); + expect(docs.find(d => d._id === 'bulk1').value).to.equal(10); + }); + }); + + // ============================================ + // SECTION 8: findOneAndUpdate + // ============================================ + + describe('findOneAndUpdate', function() { + beforeEach(async function() { + await db.collection('test').insertOne({ + _id: 'fau1', + value: 1, + name: 'original' + }); + }); + + it('should update and return the document', async function() { + const result = await db.collection('test').findOneAndUpdate( + { _id: 'fau1' }, + { $set: { name: 'updated' } }, + { returnDocument: 'after' } + ); + expect(result._id).to.equal('fau1'); + expect(result.name).to.equal('updated'); + }); + + it('should return original by default', async function() { + const result = await db.collection('test').findOneAndUpdate( + { _id: 'fau1' }, + { $set: { name: 'updated' } } + ); + expect(result._id).to.equal('fau1'); + expect(result.name).to.equal('original'); + }); + + it('should support upsert', async function() { + const result = await db.collection('test').findOneAndUpdate( + { _id: 'fau2' }, + { $set: { name: 'new' } }, + { + upsert: true, + returnDocument: 'after' + } + ); + expect(result._id).to.equal('fau2'); + expect(result.name).to.equal('new'); + }); + }); + + // ============================================ + // SECTION 9: Database Operations + // ============================================ + + describe('Database Operations', function() { + it('should get collection reference', function() { + const collection = db.collection('newcollection'); + expect(collection).to.exist; + expect(collection.collectionName || collection.name).to.equal('newcollection'); + }); + + it('should list collections', async function() { + await db.collection('listtest1').insertOne({ _id: '1' }); + await db.collection('listtest2').insertOne({ _id: '2' }); + + const collections = await db.listCollections().toArray(); + const names = collections.map(c => c.name); + expect(names).to.include('listtest1'); + expect(names).to.include('listtest2'); + }); + + it('should drop collection', async function() { + await db.collection('dropme').insertOne({ _id: '1' }); + + let collections = await db.listCollections().toArray(); + expect(collections.map(c => c.name)).to.include('dropme'); + + await db.collection('dropme').drop(); + + collections = await db.listCollections().toArray(); + expect(collections.map(c => c.name)).to.not.include('dropme'); + }); + + it('should rename collection', async function() { + await db.collection('oldname').insertOne({ + _id: 'rename1', + value: 42 + }); + + await db.collection('oldname').rename('newname'); + + const doc = await db.collection('newname').findOne({ _id: 'rename1' }); + expect(doc.value).to.equal(42); + + const oldDoc = await db.collection('oldname').findOne({ _id: 'rename1' }); + expect(oldDoc).to.be.null; + }); + }); + + // ============================================ + // SECTION 10: Nested Field Queries + // ============================================ + + describe('Nested Field Queries', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'n1', + user: { + name: 'Alice', + role: 'admin' + }, + metadata: { views: 100 } + }, + { + _id: 'n2', + user: { + name: 'Bob', + role: 'user' + }, + metadata: { views: 50 } + }, + { + _id: 'n3', + user: { + name: 'Carol', + role: 'admin' + }, + metadata: { views: 200 } + } + ]); + }); + + it('should query nested fields with dot notation', async function() { + const docs = await db.collection('test').find({ 'user.role': 'admin' }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + + it('should update nested fields with dot notation', async function() { + await db.collection('test').updateOne( + { _id: 'n1' }, + { $set: { 'user.name': 'Alicia' } } + ); + const doc = await db.collection('test').findOne({ _id: 'n1' }); + expect(doc.user.name).to.equal('Alicia'); + expect(doc.user.role).to.equal('admin'); + }); + + it('should project nested fields', async function() { + const doc = await db.collection('test').findOne( + { _id: 'n1' }, + { projection: { 'user.name': 1 } } + ); + expect(doc._id).to.equal('n1'); + expect(doc.user.name).to.equal('Alice'); + expect(doc.user.role).to.be.undefined; + expect(doc.metadata).to.be.undefined; + }); + }); + + // ============================================ + // SECTION 11: Sort on Multiple Fields + // ============================================ + + describe('Multi-field Sort', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 's1', + category: 'a', + priority: 2 + }, + { + _id: 's2', + category: 'b', + priority: 1 + }, + { + _id: 's3', + category: 'a', + priority: 1 + }, + { + _id: 's4', + category: 'b', + priority: 2 + } + ]); + }); + + it('should sort by multiple fields', async function() { + const docs = await db.collection('test') + .find({}) + .sort({ + category: 1, + priority: -1 + }) + .toArray(); + + expect(docs[0]._id).to.equal('s1'); // a, 2 + expect(docs[1]._id).to.equal('s3'); // a, 1 + expect(docs[2]._id).to.equal('s4'); // b, 2 + expect(docs[3]._id).to.equal('s2'); // b, 1 + }); + }); + + // ============================================ + // SECTION 12: Database Switching + // ============================================ + + describe('Database Switching', function() { + if (ADAPTER === 'postgres') { + it('should return the same db for nullish or matching name', function() { + const db1 = client.db(); + const db2 = client.db(); + const db3 = client.db('dbtest_adapter'); + expect(db1).to.equal(db2); + expect(db1).to.equal(db3); + }); + + it('should throw when requesting a different database name', function() { + expect(() => client.db('other_name')).to.throw(/multipostgres/); + }); + } else if (ADAPTER === 'multipostgres') { + it('should switch to sibling schema', async function() { + const siblingDb = client.db('dbtest_adapter-siblingschema'); + + await siblingDb.collection('siblingcol').insertOne({ + _id: 'sib1', + from: 'sibling' + }); + + // Verify it's not in original schema + const origDoc = await db.collection('siblingcol').findOne({ _id: 'sib1' }); + expect(origDoc).to.be.null; + + // Verify it's in sibling schema + const sibDoc = await siblingDb.collection('siblingcol').findOne({ _id: 'sib1' }); + expect(sibDoc).to.exist; + expect(sibDoc.from).to.equal('sibling'); + + // Clean up sibling + await siblingDb.dropDatabase(); + }); + } else { + it('should switch to sibling database', async function() { + const siblingDb = client.db('dbtest-sibling'); + + await siblingDb.collection('siblingcol').insertOne({ + _id: 'sib1', + from: 'sibling' + }); + + // Verify it's not in original + const origDoc = await db.collection('siblingcol').findOne({ _id: 'sib1' }); + expect(origDoc).to.be.null; + + // Verify it's in sibling + const sibDoc = await siblingDb.collection('siblingcol').findOne({ _id: 'sib1' }); + expect(sibDoc).to.exist; + expect(sibDoc.from).to.equal('sibling'); + + // Clean up sibling + await siblingDb.collection('siblingcol').drop(); + }); + } + }); + + // ============================================ + // SECTION 13: Empty Results + // ============================================ + + describe('Empty Results Handling', function() { + it('should return empty array for find with no matches', async function() { + const docs = await db.collection('test').find({ nonexistent: true }).toArray(); + expect(docs).to.be.an('array').that.is.empty; + }); + + it('should return 0 for count with no matches', async function() { + const count = await db.collection('test').countDocuments({ nonexistent: true }); + expect(count).to.equal(0); + }); + + it('should return empty array for distinct with no matches', async function() { + const values = await db.collection('test').distinct('field', { nonexistent: true }); + expect(values).to.be.an('array').that.is.empty; + }); + }); + + // ============================================ + // SECTION 14: Date Handling + // ============================================ + + describe('Date Handling', function() { + it('should store and retrieve Date objects', async function() { + const now = new Date(); + await db.collection('test').insertOne({ + _id: 'date1', + createdAt: now + }); + + const doc = await db.collection('test').findOne({ _id: 'date1' }); + expect(doc.createdAt).to.be.instanceOf(Date); + expect(doc.createdAt.getTime()).to.equal(now.getTime()); + }); + + it('should query by date comparison', async function() { + const old = new Date('2020-01-01'); + const recent = new Date('2024-01-01'); + const cutoff = new Date('2022-01-01'); + + await db.collection('test').insertMany([ + { + _id: 'old', + createdAt: old + }, + { + _id: 'recent', + createdAt: recent + } + ]); + + const docs = await db.collection('test').find({ createdAt: { $gte: cutoff } }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('recent'); + }); + }); + + // ============================================ + // SECTION 15: Null and Undefined Handling + // ============================================ + + describe('Null and Undefined Handling', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'null1', + value: null + }, + { + _id: 'null2', + value: 'present' + }, + { _id: 'null3' } // value field missing + ]); + }); + + it('should find documents with null value', async function() { + const docs = await db.collection('test').find({ value: null }).toArray(); + // MongoDB matches both explicit null AND missing fields with + // { value: null }, so null1 (explicit null) and null3 (missing) match, + // but null2 (value: 'present') does not. + expect(docs).to.have.lengthOf(2); + expect(docs.map(d => d._id).sort()).to.deep.equal([ 'null1', 'null3' ]); + }); + + it('should distinguish null from missing with $exists', async function() { + const withField = await db.collection('test').find({ value: { $exists: true } }).toArray(); + expect(withField).to.have.lengthOf(2); + + const withoutField = await db.collection('test').find({ value: { $exists: false } }).toArray(); + expect(withoutField).to.have.lengthOf(1); + expect(withoutField[0]._id).to.equal('null3'); + }); + }); + + // ============================================ + // SECTION 16: Mixed Type Arrays + // ============================================ + + describe('Mixed Type Arrays', function() { + it('should handle arrays with mixed types', async function() { + await db.collection('test').insertOne({ + _id: 'mixed1', + items: [ 1, 'two', { three: 3 }, [ 4, 5 ], null ] + }); + + const doc = await db.collection('test').findOne({ _id: 'mixed1' }); + expect(doc.items).to.deep.equal([ 1, 'two', { three: 3 }, [ 4, 5 ], null ]); + }); + }); + + // ============================================ + // SECTION 17: Large Documents + // ============================================ + + describe('Large Documents', function() { + it('should handle documents with many fields', async function() { + const doc = { _id: 'large1' }; + for (let i = 0; i < 100; i++) { + doc[`field${i}`] = `value${i}`; + } + + await db.collection('test').insertOne(doc); + const retrieved = await db.collection('test').findOne({ _id: 'large1' }); + + expect(retrieved.field0).to.equal('value0'); + expect(retrieved.field99).to.equal('value99'); + }); + + it('should handle large string values', async function() { + const largeString = 'x'.repeat(100000); + await db.collection('test').insertOne({ + _id: 'largestr', + content: largeString + }); + + const doc = await db.collection('test').findOne({ _id: 'largestr' }); + expect(doc.content).to.equal(largeString); + }); + }); + + // ============================================ + // SECTION 18: Multiple Update Operators + // ============================================ + + describe('Multiple Update Operators Combined', function() { + it('should apply multiple update operators in single update', async function() { + await db.collection('test').insertOne({ + _id: 'multi1', + count: 5, + name: 'original', + tags: [ 'a' ], + toRemove: 'value' + }); + + await db.collection('test').updateOne( + { _id: 'multi1' }, + { + $set: { name: 'updated' }, + $inc: { count: 3 }, + $push: { tags: 'b' }, + $unset: { toRemove: '' } + } + ); + + const doc = await db.collection('test').findOne({ _id: 'multi1' }); + expect(doc.name).to.equal('updated'); + expect(doc.count).to.equal(8); + expect(doc.tags).to.deep.equal([ 'a', 'b' ]); + expect(doc.toRemove).to.be.undefined; + }); + }); + + // ============================================ + // SECTION 19: Atomicity + // ============================================ + + describe('Atomicity', function() { + it('should ensure atomic _id uniqueness', async function() { + // Run multiple concurrent inserts with same _id + const promises = []; + for (let i = 0; i < 10; i++) { + promises.push( + db.collection('test').insertOne({ + _id: 'atomic1', + value: i + }) + .then(() => 'success') + .catch(() => 'duplicate') + ); + } + + const results = await Promise.all(promises); + const successes = results.filter(r => r === 'success'); + const duplicates = results.filter(r => r === 'duplicate'); + + // Exactly one should succeed + expect(successes).to.have.lengthOf(1); + expect(duplicates).to.have.lengthOf(9); + + // Verify only one document exists + const count = await db.collection('test').countDocuments({ _id: 'atomic1' }); + expect(count).to.equal(1); + }); + }); + + // ============================================ + // SECTION 20: Type Preservation + // ============================================ + + describe('Type Preservation', function() { + it('should preserve JavaScript types', async function() { + const testDoc = { + _id: 'types1', + string: 'hello', + number: 42, + float: 3.14159, + boolean: true, + date: new Date('2024-01-15T12:00:00Z'), + array: [ 1, 2, 3 ], + nested: { + a: 1, + b: { c: 2 } + }, + nullValue: null + }; + + await db.collection('test').insertOne(testDoc); + const doc = await db.collection('test').findOne({ _id: 'types1' }); + + expect(typeof doc.string).to.equal('string'); + expect(typeof doc.number).to.equal('number'); + expect(typeof doc.float).to.equal('number'); + expect(typeof doc.boolean).to.equal('boolean'); + expect(doc.date).to.be.instanceOf(Date); + expect(Array.isArray(doc.array)).to.be.true; + expect(typeof doc.nested).to.equal('object'); + expect(doc.nullValue).to.be.null; + }); + }); + + // ============================================ + // SECTION 21: Multi-schema Mode (multipostgres only) + // ============================================ + + // ============================================ + // SECTION 22: Batched Containment Queries + // ============================================ + + describe('Batched Containment Queries', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'bc1', + type: 'article', + slug: '/news', + status: 'published', + tags: [ 'featured', 'news' ] + }, + { + _id: 'bc2', + type: 'article', + slug: '/blog', + status: 'draft', + tags: [ 'blog' ] + }, + { + _id: 'bc3', + type: 'page', + slug: '/home', + status: 'published', + tags: [ 'featured' ] + }, + { + _id: 'bc4', + type: 'page', + slug: '/about', + status: 'published', + tags: [] + } + ]); + }); + + it('should match multi-field scalar equality', async function() { + const docs = await db.collection('test').find({ + type: 'article', + status: 'published' + }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('bc1'); + }); + + it('should match three-field scalar equality', async function() { + const docs = await db.collection('test').find({ + type: 'page', + status: 'published', + slug: '/home' + }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('bc3'); + }); + + it('should match scalar value in array field via containment', async function() { + // { tags: 'featured' } matches docs where + // tags contains 'featured' + const docs = await db.collection('test').find({ + tags: 'featured', + type: 'article' + }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('bc1'); + }); + + it('should handle nested field equality in containment', async function() { + await db.collection('test').insertOne({ + _id: 'bc5', + user: { + profile: { + name: 'Alice', + role: 'admin' + } + } + }); + await db.collection('test').insertOne({ + _id: 'bc6', + user: { + profile: { + name: 'Bob', + role: 'admin' + } + } + }); + const docs = await db.collection('test').find({ + 'user.profile.role': 'admin', + 'user.profile.name': 'Alice' + }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('bc5'); + }); + + it('should combine containment with operator conditions', async function() { + // Mix of scalar equality (batched) and operator conditions + const docs = await db.collection('test').find({ + type: 'page', + status: { $ne: 'draft' } + }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + + it('should handle boolean values in containment', async function() { + await db.collection('test').insertOne({ + _id: 'bc7', + active: true, + visible: false + }); + const docs = await db.collection('test').find({ + active: true, + visible: false + }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('bc7'); + }); + + it('should handle numeric values in containment', async function() { + await db.collection('test').insertOne({ + _id: 'bc8', + level: 5, + score: 100 + }); + const docs = await db.collection('test').find({ + level: 5, + score: 100 + }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('bc8'); + }); + + it('should handle Date values in containment', async function() { + const date = new Date('2024-06-15T12:00:00Z'); + await db.collection('test').insertOne({ + _id: 'bc9', + createdAt: date, + type: 'event' + }); + const docs = await db.collection('test').find({ + type: 'event', + createdAt: date + }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('bc9'); + expect(docs[0].createdAt).to.be.instanceOf(Date); + expect(docs[0].createdAt.getTime()).to.equal(date.getTime()); + }); + }); + + // ============================================ + // SECTION 23: $in on _id Field + // ============================================ + + describe('$in on _id field', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'in1', + value: 1 + }, + { + _id: 'in2', + value: 2 + }, + { + _id: 'in3', + value: 3 + }, + { + _id: 'in4', + value: 4 + }, + { + _id: 'in5', + value: 5 + } + ]); + }); + + it('should find documents by _id $in', async function() { + const docs = await db.collection('test').find({ + _id: { $in: [ 'in1', 'in3', 'in5' ] } + }).sort({ _id: 1 }).toArray(); + expect(docs).to.have.lengthOf(3); + expect(docs[0]._id).to.equal('in1'); + expect(docs[1]._id).to.equal('in3'); + expect(docs[2]._id).to.equal('in5'); + }); + + it('should handle _id $in with single value', async function() { + const docs = await db.collection('test').find({ + _id: { $in: [ 'in2' ] } + }).toArray(); + expect(docs).to.have.lengthOf(1); + expect(docs[0]._id).to.equal('in2'); + }); + + it('should handle _id $in with empty array', async function() { + const docs = await db.collection('test').find({ + _id: { $in: [] } + }).toArray(); + expect(docs).to.have.lengthOf(0); + }); + + it('should handle _id $in with many values', async function() { + // Test with more IDs than exist — should only return matching ones + const ids = []; + for (let i = 1; i <= 20; i++) { + ids.push(`in${i}`); + } + const docs = await db.collection('test').find({ + _id: { $in: ids } + }).toArray(); + expect(docs).to.have.lengthOf(5); + }); + + it('should handle _id $in with non-matching values', async function() { + const docs = await db.collection('test').find({ + _id: { $in: [ 'nonexistent1', 'nonexistent2' ] } + }).toArray(); + expect(docs).to.have.lengthOf(0); + }); + + it('should combine _id $in with other conditions', async function() { + const docs = await db.collection('test').find({ + _id: { $in: [ 'in1', 'in2', 'in3' ] }, + value: { $gt: 1 } + }).toArray(); + expect(docs).to.have.lengthOf(2); + }); + }); + + // ============================================ + // SECTION 24: $in on Array Fields + // ============================================ + + describe('$in on array fields', function() { + beforeEach(async function() { + await db.collection('test').insertMany([ + { + _id: 'ia1', + name: 'Alice', + roles: [ 'admin', 'editor' ] + }, + { + _id: 'ia2', + name: 'Bob', + roles: [ 'viewer' ] + }, + { + _id: 'ia3', + name: 'Carol', + roles: [ 'editor', 'viewer' ] + }, + { + _id: 'ia4', + name: 'Dave', + roles: [ 'admin' ] + } + ]); + }); + + it('$in should match array field containing any of the values', async function() { + const docs = await db.collection('test').find({ + roles: { $in: [ 'admin', 'viewer' ] } + }).sort({ _id: 1 }).toArray(); + // ia1 has admin, ia2 has viewer, ia3 has viewer, ia4 has admin + expect(docs).to.have.lengthOf(4); + expect(docs.map(d => d._id)).to.deep.equal([ 'ia1', 'ia2', 'ia3', 'ia4' ]); + }); + + it('$in should match scalar field normally', async function() { + const docs = await db.collection('test').find({ + name: { $in: [ 'Alice', 'Carol' ] } + }).sort({ _id: 1 }).toArray(); + expect(docs).to.have.lengthOf(2); + expect(docs.map(d => d._id)).to.deep.equal([ 'ia1', 'ia3' ]); + }); + + it('$in with null should match missing fields', async function() { + await db.collection('test').insertOne({ + _id: 'ia5', + name: 'Eve' + // no roles field + }); + const docs = await db.collection('test').find({ + roles: { $in: [ 'admin', null ] } + }).sort({ _id: 1 }).toArray(); + // Should match ia1 (has admin), ia4 (has admin), and ia5 (roles missing) + expect(docs).to.have.lengthOf(3); + expect(docs.map(d => d._id)).to.deep.equal([ 'ia1', 'ia4', 'ia5' ]); + }); + }); + + // ============================================ + // SECTION 25: Deserialization Optimization + // ============================================ + + describe('Deserialization Optimization', function() { + it('should correctly roundtrip documents with no dates', async function() { + const original = { + _id: 'deser1', + title: 'No Dates Here', + count: 42, + active: true, + tags: [ 'a', 'b', 'c' ], + nested: { + deep: { + value: 'hello', + list: [ 1, 2, 3 ] + } + } + }; + await db.collection('test').insertOne(original); + const doc = await db.collection('test').findOne({ _id: 'deser1' }); + expect(doc._id).to.equal('deser1'); + expect(doc.title).to.equal('No Dates Here'); + expect(doc.count).to.equal(42); + expect(doc.active).to.equal(true); + expect(doc.tags).to.deep.equal([ 'a', 'b', 'c' ]); + expect(doc.nested.deep.value).to.equal('hello'); + expect(doc.nested.deep.list).to.deep.equal([ 1, 2, 3 ]); + }); + + it('should correctly roundtrip documents with deeply nested dates', async function() { + const date1 = new Date('2024-01-15T10:30:00Z'); + const date2 = new Date('2024-06-01T00:00:00Z'); + const original = { + _id: 'deser2', + title: 'Has Dates', + metadata: { + createdAt: date1, + nested: { + modifiedAt: date2, + plain: 'no date here' + } + }, + tags: [ 'a', 'b' ] + }; + await db.collection('test').insertOne(original); + const doc = await db.collection('test').findOne({ _id: 'deser2' }); + expect(doc.metadata.createdAt).to.be.instanceOf(Date); + expect(doc.metadata.createdAt.getTime()).to.equal(date1.getTime()); + expect(doc.metadata.nested.modifiedAt).to.be.instanceOf(Date); + expect(doc.metadata.nested.modifiedAt.getTime()).to.equal(date2.getTime()); + expect(doc.metadata.nested.plain).to.equal('no date here'); + expect(doc.tags).to.deep.equal([ 'a', 'b' ]); + }); + + it('should correctly roundtrip documents with dates in arrays', async function() { + const date1 = new Date('2024-03-01T00:00:00Z'); + const date2 = new Date('2024-04-01T00:00:00Z'); + const original = { + _id: 'deser3', + events: [ + { + name: 'event1', + date: date1 + }, + { + name: 'event2', + date: date2 + }, + { name: 'event3' } // no date + ] + }; + await db.collection('test').insertOne(original); + const doc = await db.collection('test').findOne({ _id: 'deser3' }); + expect(doc.events[0].date).to.be.instanceOf(Date); + expect(doc.events[0].date.getTime()).to.equal(date1.getTime()); + expect(doc.events[1].date).to.be.instanceOf(Date); + expect(doc.events[1].date.getTime()).to.equal(date2.getTime()); + expect(doc.events[2].date).to.be.undefined; + }); + + it('should handle multiple documents efficiently via find', async function() { + // Insert mix of documents with and without dates + await db.collection('test').insertMany([ + { + _id: 'deser4', + title: 'Plain', + value: 1 + }, + { + _id: 'deser5', + title: 'With Date', + createdAt: new Date('2024-01-01') + }, + { + _id: 'deser6', + title: 'Also Plain', + value: 3 + } + ]); + const docs = await db.collection('test').find({ + _id: { $in: [ 'deser4', 'deser5', 'deser6' ] } + }).sort({ _id: 1 }).toArray(); + expect(docs).to.have.lengthOf(3); + expect(docs[0].title).to.equal('Plain'); + expect(docs[0].createdAt).to.be.undefined; + expect(docs[1].createdAt).to.be.instanceOf(Date); + expect(docs[2].title).to.equal('Also Plain'); + }); + }); + + // ============================================ + // SECTION 26: Full-Text Search + // ============================================ + + describe('Full-Text Search', function() { + beforeEach(async function() { + const col = db.collection('search'); + try { + await col.drop(); + } catch (e) { /* ignore */ } + // Create a text index on title and body + await col.createIndex({ + title: 'text', + body: 'text' + }); + // Insert documents with varying relevance to "database migration" + await col.insertMany([ + { + _id: 'full-match', + title: 'Database Migration Guide', + body: 'This guide covers database migration strategies for production systems. Database migration is critical.' + }, + { + _id: 'title-only', + title: 'Database Basics', + body: 'An introduction to storing and retrieving information.' + }, + { + _id: 'body-only', + title: 'System Administration', + body: 'Learn about database backup and migration procedures.' + }, + { + _id: 'no-match', + title: 'Cooking Recipes', + body: 'How to make a perfect sourdough bread.' + } + ]); + }); + + afterEach(async function() { + try { + await db.collection('search').drop(); + } catch (e) { /* ignore */ } + }); + + it('$text should match documents containing search terms', async function() { + const results = await db.collection('search') + .find({ $text: { $search: 'database' } }) + .toArray(); + const ids = results.map(d => d._id); + expect(ids).to.include('full-match'); + expect(ids).to.include('title-only'); + expect(ids).to.include('body-only'); + expect(ids).to.not.include('no-match'); + }); + + it('$text should combine with other query operators', async function() { + const results = await db.collection('search') + .find({ + $text: { $search: 'database' }, + _id: { $ne: 'title-only' } + }) + .toArray(); + const ids = results.map(d => d._id); + expect(ids).to.include('full-match'); + expect(ids).to.include('body-only'); + expect(ids).to.not.include('title-only'); + expect(ids).to.not.include('no-match'); + }); + + it('$text with empty search should match nothing', async function() { + const results = await db.collection('search') + .find({ $text: { $search: ' ' } }) + .toArray(); + expect(results).to.have.lengthOf(0); + }); + + it('$text should use fields from the text index, not hardcoded defaults', async function() { + // The text index is on title and body. A search for content + // in those fields should work. A value only present in an + // un-indexed field should not match. + const col = db.collection('search'); + await col.insertOne({ + _id: 'unindexed', + title: 'Nothing special', + body: 'Nothing special', + notes: 'database migration' + }); + const results = await col + .find({ $text: { $search: 'sourdough' } }) + .toArray(); + // 'sourdough' appears only in no-match's body, which IS indexed + expect(results.map(d => d._id)).to.include('no-match'); + // 'unindexed' has 'database migration' only in notes (not indexed) + // so a search that only matches notes should not find it + const results2 = await col + .find({ + $text: { $search: 'database' }, + _id: 'unindexed' + }) + .toArray(); + expect(results2).to.have.lengthOf(0); + }); + + it('should rank results by relevance when sorted by textScore', async function() { + const results = await db.collection('search') + .find({ $text: { $search: 'database migration' } }) + .sort({ score: { $meta: 'textScore' } }) + .project({ score: { $meta: 'textScore' } }) + .toArray(); + // full-match has "database" 2x and "migration" 2x — should rank first + expect(results.length).to.be.at.least(3); + expect(results[0]._id).to.equal('full-match'); + }); + + it('should expose textScore via $meta projection', async function() { + const results = await db.collection('search') + .find({ $text: { $search: 'database migration' } }) + .sort({ score: { $meta: 'textScore' } }) + .project({ + title: 1, + score: { $meta: 'textScore' } + }) + .toArray(); + expect(results.length).to.be.at.least(1); + // Each result should have a numeric score + for (const doc of results) { + expect(doc.score).to.be.a('number'); + expect(doc.score).to.be.greaterThan(0); + // Projection should still include requested fields + expect(doc.title).to.be.a('string'); + } + // Higher-relevance doc should have a higher score + const fullMatch = results.find(d => d._id === 'full-match'); + const titleOnly = results.find(d => d._id === 'title-only'); + if (fullMatch && titleOnly) { + expect(fullMatch.score).to.be.greaterThan(titleOnly.score); + } + }); + + it('should support sorting by $meta textScore', async function() { + const results = await db.collection('search') + .find({ $text: { $search: 'database migration' } }) + .sort({ score: { $meta: 'textScore' } }) + .project({ score: { $meta: 'textScore' } }) + .toArray(); + expect(results.length).to.be.at.least(2); + // Scores should be in descending order + for (let i = 1; i < results.length; i++) { + expect(results[i - 1].score).to.be.at.least(results[i].score); + } + }); + + it('should sort by relevance with textScore among other sort fields', async function() { + // Insert additional docs to make ranking clearer + const col = db.collection('search'); + await col.insertOne({ + _id: 'weak-match', + title: 'Random Notes', + body: 'Contains the word database once among other unrelated content about gardening and weather.' + }); + + const results = await col + .find({ $text: { $search: 'database migration' } }) + .sort({ score: { $meta: 'textScore' } }) + .project({ score: { $meta: 'textScore' } }) + .toArray(); + + expect(results.length).to.be.at.least(2); + // Scores should be in descending order + for (let i = 1; i < results.length; i++) { + expect(results[i - 1].score).to.be.at.least(results[i].score); + } + }); + }); // Full-Text Search + + if (ADAPTER === 'multipostgres') { + describe('Multi-schema Mode', function() { + it('should store tables in the named schema, not public', async function() { + // Insert a doc to ensure the table exists in the schema + await db.collection('schematest').insertOne({ + _id: 'st1', + value: 'hello' + }); + + // Check that the table exists in the named schema + const { Pool } = require('pg'); + const pool = new Pool({ connectionString: 'postgres://localhost:5432/dbtest_adapter' }); + try { + const inSchema = await pool.query( + 'SELECT tablename FROM pg_tables WHERE schemaname = \'testschema\' AND tablename = \'schematest\'' + ); + expect(inSchema.rows).to.have.lengthOf(1); + + const inPublic = await pool.query( + 'SELECT tablename FROM pg_tables WHERE schemaname = \'public\' AND tablename = \'schematest\'' + ); + expect(inPublic.rows).to.have.lengthOf(0); + } finally { + await pool.end(); + } + + // Clean up + await db.collection('schematest').drop(); + }); + + it('should list schemas as databases via admin().listDatabases()', async function() { + // Ensure at least one table exists so the schema is created + await db.collection('admintest').insertOne({ + _id: 'at1', + value: 1 + }); + + const result = await db.admin().listDatabases(); + expect(result.databases).to.be.an('array'); + const names = result.databases.map(d => d.name); + expect(names).to.include('dbtest_adapter-testschema'); + + await db.collection('admintest').drop(); + }); + + it('should drop schema via dropDatabase()', async function() { + const tempDb = client.db('dbtest_adapter-dropschematest'); + await tempDb.collection('tempcol').insertOne({ + _id: 'tmp1', + value: 1 + }); + + // Verify schema exists + const { Pool } = require('pg'); + const pool = new Pool({ connectionString: 'postgres://localhost:5432/dbtest_adapter' }); + try { + let schemas = await pool.query( + 'SELECT schema_name FROM information_schema.schemata WHERE schema_name = \'dropschematest\'' + ); + expect(schemas.rows).to.have.lengthOf(1); + + // Drop it + await tempDb.dropDatabase(); + + schemas = await pool.query( + 'SELECT schema_name FROM information_schema.schemata WHERE schema_name = \'dropschematest\'' + ); + expect(schemas.rows).to.have.lengthOf(0); + } finally { + await pool.end(); + } + }); + }); + } +}); diff --git a/packages/db-connect/test/dump-restore.test.js b/packages/db-connect/test/dump-restore.test.js new file mode 100644 index 0000000000..a0c0835b3c --- /dev/null +++ b/packages/db-connect/test/dump-restore.test.js @@ -0,0 +1,180 @@ +/* global describe, it, before, after */ +const { expect } = require('chai'); +const dbConnect = require('..'); + +const ADAPTER = process.env.ADAPTER || 'mongodb'; + +function getUri(dbName) { + if (ADAPTER === 'mongodb') { + return `mongodb://localhost:27017/${dbName}`; + } else if (ADAPTER === 'postgres') { + const user = process.env.PGUSER || process.env.USER; + const password = process.env.PGPASSWORD || ''; + const auth = password ? `${user}:${password}@` : `${user}@`; + return `postgres://${auth}localhost:5432/${dbName}`; + } else if (ADAPTER === 'multipostgres') { + const user = process.env.PGUSER || process.env.USER; + const password = process.env.PGPASSWORD || ''; + const auth = password ? `${user}:${password}@` : `${user}@`; + return `multipostgres://${auth}localhost:5432/dbtest_dump-${dbName}`; + } else if (ADAPTER === 'sqlite') { + return `sqlite:///tmp/${dbName}.db`; + } + throw new Error(`Unknown adapter: ${ADAPTER}`); +} + +describe(`dump/restore programmatic API (${ADAPTER})`, function () { + const sourceDbName = 'dbtest_dump_source'; + const targetDbName = 'dbtest_dump_target'; + let sourceClient; + let targetClient; + + before(async function () { + // Clean up any previous runs + sourceClient = await dbConnect(getUri(sourceDbName)); + targetClient = await dbConnect(getUri(targetDbName)); + const sourceDb = sourceClient.db(); + const targetDb = targetClient.db(); + try { + await sourceDb.collection('items').drop(); + } catch (e) { /* ignore */ } + try { + await sourceDb.collection('meta').drop(); + } catch (e) { /* ignore */ } + try { + await targetDb.collection('items').drop(); + } catch (e) { /* ignore */ } + try { + await targetDb.collection('meta').drop(); + } catch (e) { /* ignore */ } + await sourceClient.close(); + await targetClient.close(); + }); + + after(async function () { + // Clean up + sourceClient = await dbConnect(getUri(sourceDbName)); + targetClient = await dbConnect(getUri(targetDbName)); + try { + await sourceClient.db().collection('items').drop(); + } catch (e) { /* ignore */ } + try { + await sourceClient.db().collection('meta').drop(); + } catch (e) { /* ignore */ } + try { + await targetClient.db().collection('items').drop(); + } catch (e) { /* ignore */ } + try { + await targetClient.db().collection('meta').drop(); + } catch (e) { /* ignore */ } + await sourceClient.close(); + await targetClient.close(); + }); + + it('dump yields an async iterable of NDJSON lines', async function () { + // Insert some data + const client = await dbConnect(getUri(sourceDbName)); + const db = client.db(); + await db.collection('items').insertMany([ + { + _id: 'item1', + title: 'First', + value: 10 + }, + { + _id: 'item2', + title: 'Second', + value: 20 + } + ]); + await db.collection('meta').insertOne({ + _id: 'version', + v: 1 + }); + await client.close(); + + const iter = dbConnect.dump(getUri(sourceDbName)); + // Must be an async iterable — NOT a string (API contract: dump is + // streamed so large databases never sit fully in memory). + expect(iter).to.not.be.a('string'); + expect(typeof iter[Symbol.asyncIterator]).to.equal('function'); + + const lines = []; + for await (const line of iter) { + lines.push(line); + } + // Every emitted record must be a non-empty single JSON object — + // no embedded newlines, parseable as JSON. + for (const line of lines) { + expect(line).to.be.a('string'); + expect(line).to.not.include('\n'); + expect(() => JSON.parse(line)).to.not.throw(); + } + const joined = lines.join('\n'); + expect(joined).to.include('item1'); + expect(joined).to.include('First'); + expect(joined).to.include('item2'); + expect(joined).to.include('version'); + }); + + it('should restore a database from a dump stream', async function () { + await dbConnect.restore(getUri(targetDbName), dbConnect.dump(getUri(sourceDbName))); + + const client = await dbConnect(getUri(targetDbName)); + const db = client.db(); + const items = await db.collection('items').find({}).sort({ _id: 1 }).toArray(); + expect(items).to.have.length(2); + expect(items[0]._id).to.equal('item1'); + expect(items[0].title).to.equal('First'); + expect(items[1]._id).to.equal('item2'); + expect(items[1].title).to.equal('Second'); + + const meta = await db.collection('meta').findOne({ _id: 'version' }); + expect(meta.v).to.equal(1); + await client.close(); + }); + + it('should copy a database via copyDatabase()', async function () { + // Modify source to prove we get fresh data + const client = await dbConnect(getUri(sourceDbName)); + await client.db().collection('items').insertOne({ + _id: 'item3', + title: 'Third', + value: 30 + }); + await client.close(); + + // Clean target first + const tgt = await dbConnect(getUri(targetDbName)); + try { + await tgt.db().collection('items').drop(); + } catch (e) { /* ignore */ } + try { + await tgt.db().collection('meta').drop(); + } catch (e) { /* ignore */ } + await tgt.close(); + + await dbConnect.copyDatabase(getUri(sourceDbName), getUri(targetDbName)); + + const check = await dbConnect(getUri(targetDbName)); + const items = await check.db().collection('items').find({}).sort({ _id: 1 }).toArray(); + expect(items).to.have.length(3); + expect(items[2].title).to.equal('Third'); + await check.close(); + }); + + it('should produce independent databases after copy', async function () { + await dbConnect.copyDatabase(getUri(sourceDbName), getUri(targetDbName)); + + // Modify target + const tgt = await dbConnect(getUri(targetDbName)); + await tgt.db().collection('items').updateOne({ _id: 'item1' }, { $set: { title: 'Modified' } }); + await tgt.close(); + + // Source should be unchanged + const src = await dbConnect(getUri(sourceDbName)); + const item = await src.db().collection('items').findOne({ _id: 'item1' }); + expect(item.title).to.equal('First'); + await src.close(); + }); +}); diff --git a/packages/db-connect/test/security.test.js b/packages/db-connect/test/security.test.js new file mode 100644 index 0000000000..399b16dbed --- /dev/null +++ b/packages/db-connect/test/security.test.js @@ -0,0 +1,316 @@ +/* global describe, it, before, after, beforeEach */ +/* eslint-disable no-unused-expressions */ +const { expect } = require('chai'); + +// Security tests for SQL injection prevention +// These tests verify that inputs are properly escaped or rejected as appropriate + +const ADAPTER = process.env.ADAPTER || 'mongodb'; + +describe(`Security Tests (${ADAPTER})`, function() { + let client; + let db; + + before(async function() { + if (ADAPTER === 'mongodb') { + const mongodb = require('../adapters/mongodb'); + client = await mongodb.connect('mongodb://localhost:27017/dbtest-security'); + db = client.db(); + } else if (ADAPTER === 'postgres') { + const postgres = require('../adapters/postgres'); + const user = process.env.PGUSER || process.env.USER; + const password = process.env.PGPASSWORD || ''; + const auth = password ? `${user}:${password}@` : `${user}@`; + client = await postgres.connect(`postgres://${auth}localhost:5432/dbtest_adapter`); + db = client.db('dbtest_adapter'); + } else if (ADAPTER === 'multipostgres') { + const postgres = require('../adapters/postgres'); + const user = process.env.PGUSER || process.env.USER; + const password = process.env.PGPASSWORD || ''; + const auth = password ? `${user}:${password}@` : `${user}@`; + client = await postgres.connect(`multipostgres://${auth}localhost:5432/dbtest_adapter-securitytest`); + db = client.db(); + } else if (ADAPTER === 'sqlite') { + const sqlite = require('../adapters/sqlite'); + const os = require('os'); + const pathModule = require('path'); + const fs = require('fs'); + const dbPath = pathModule.join(os.tmpdir(), 'dbtest-security.db'); + try { + fs.unlinkSync(dbPath); + } catch (e) { /* ignore */ } + client = await sqlite.connect(`sqlite://${dbPath}`); + db = client.db(); + } + }); + + after(async function() { + if (db) { + try { + await db.collection('sectest').drop(); + } catch (e) { + // ignore + } + } + if (client) { + await client.close(); + } + }); + + // These tests only apply to postgres adapter since MongoDB doesn't have SQL + if (ADAPTER === 'postgres' || ADAPTER === 'sqlite') { + describe('SQL Injection Prevention', function() { + describe('Field Names (escaping, not rejection)', function() { + it('should safely escape field names with single quotes', async function() { + // Field names with quotes should be escaped, not cause SQL injection + await db.collection('sectest').insertOne({ + _id: 'esc1', + 'field\'test': 'value' + }); + const doc = await db.collection('sectest').findOne({ 'field\'test': 'value' }); + expect(doc).to.exist; + expect(doc['field\'test']).to.equal('value'); + await db.collection('sectest').deleteOne({ _id: 'esc1' }); + }); + + it('should safely escape field names with SQL-like content', async function() { + // This should NOT execute SQL injection, just be a weird field name + await db.collection('sectest').insertOne({ + _id: 'esc2', + 'field; DROP TABLE users;': 'value' + }); + const doc = await db.collection('sectest').findOne({ 'field; DROP TABLE users;': 'value' }); + expect(doc).to.exist; + await db.collection('sectest').deleteOne({ _id: 'esc2' }); + }); + + it('should safely handle field names with special characters', async function() { + await db.collection('sectest').insertOne({ + _id: 'esc3', + 'field()': 'value', + 'field"quote': 'test' + }); + const doc = await db.collection('sectest').findOne({ _id: 'esc3' }); + expect(doc).to.exist; + expect(doc['field()']).to.equal('value'); + expect(doc['field"quote']).to.equal('test'); + await db.collection('sectest').deleteOne({ _id: 'esc3' }); + }); + + it('should allow nested field names with dots', async function() { + await db.collection('sectest').insertOne({ + _id: 'sec1', + user: { name: 'test' } + }); + const doc = await db.collection('sectest').findOne({ 'user.name': 'test' }); + expect(doc).to.exist; + await db.collection('sectest').deleteOne({ _id: 'sec1' }); + }); + + it('should allow field names with hyphens', async function() { + await db.collection('sectest').insertOne({ + _id: 'sec4', + 'my-field-name': 'value' + }); + const doc = await db.collection('sectest').findOne({ 'my-field-name': 'value' }); + expect(doc).to.exist; + await db.collection('sectest').deleteOne({ _id: 'sec4' }); + }); + }); + + describe('Collection Names', function() { + it('should reject collection names with SQL injection', async function() { + try { + db.collection('test\'; DROP TABLE users;--'); + expect.fail('Should have rejected malicious collection name'); + } catch (e) { + expect(e.message).to.include('Invalid table name'); + } + }); + + it('should reject collection names with double quotes', async function() { + try { + db.collection('test"injection'); + expect.fail('Should have rejected malicious collection name'); + } catch (e) { + expect(e.message).to.include('Invalid table name'); + } + }); + + it('should allow collection names with hyphens (converted to underscores)', async function() { + const col = db.collection('test-collection'); + expect(col.name).to.equal('test-collection'); + // The internal table name should have underscores and be prefixed with db name + expect(col._tableName).to.include('test_collection'); + }); + + it('should allow standard alphanumeric collection names', async function() { + const col = db.collection('MyCollection123'); + expect(col.name).to.equal('MyCollection123'); + }); + }); + + describe('Index Names', function() { + beforeEach(async function() { + try { + await db.collection('sectest').drop(); + } catch (e) { + // ignore + } + await db.collection('sectest').insertOne({ + _id: 'idx1', + field: 'value' + }); + }); + + it('should sanitize malicious index names', async function() { + // Index names arrive from cross-backend dumps (e.g. MongoDB defaults + // like "collection._id_1") so we sanitize rather than reject. The + // security property we care about is that no dangerous characters + // reach the SQL statement — quotes, semicolons, dashes must all be + // replaced before the name is interpolated into CREATE INDEX. + const returned = await db.collection('sectest').createIndex( + { field: 1 }, + { name: 'idx\'; DROP TABLE users;--' } + ); + expect(returned).to.not.match(/['";\- ]/); + // And the table must still exist — sanity check that no injection ran. + const doc = await db.collection('sectest').findOne({ _id: 'idx1' }); + expect(doc).to.not.be.null; + }); + + it('should allow valid index names', async function() { + const indexName = await db.collection('sectest').createIndex({ field: 1 }, { name: 'my_custom_index' }); + expect(indexName).to.equal('my_custom_index'); + }); + }); + + describe('Operator Values', function() { + beforeEach(async function() { + try { + await db.collection('sectest').drop(); + } catch (e) { + // ignore + } + await db.collection('sectest').insertOne({ + _id: 'op1', + name: 'test', + count: 5 + }); + }); + + it('should safely handle malicious string values (parameterized)', async function() { + // This should not cause SQL injection because values are parameterized + const doc = await db.collection('sectest').findOne({ name: '\'; DROP TABLE users;--' }); + expect(doc).to.be.null; // No match, but no error either + }); + + it('should safely handle malicious _id values (parameterized)', async function() { + const doc = await db.collection('sectest').findOne({ _id: '\'; DROP TABLE users;--' }); + expect(doc).to.be.null; // No match, but no error either + }); + + it('should safely handle malicious values in $in operator', async function() { + const docs = await db.collection('sectest').find({ + name: { $in: [ '\'; DROP TABLE users;--', 'normal' ] } + }).toArray(); + expect(docs).to.have.lengthOf(0); + }); + + it('should safely handle malicious regex patterns', async function() { + // Regex patterns are parameterized + const docs = await db.collection('sectest').find({ + name: { $regex: '\'; DROP TABLE' } + }).toArray(); + expect(docs).to.have.lengthOf(0); + }); + }); + + describe('LIMIT and OFFSET Validation', function() { + beforeEach(async function() { + try { + await db.collection('sectest').drop(); + } catch (e) { + // ignore + } + await db.collection('sectest').insertMany([ + { + _id: 'lim1', + value: 1 + }, + { + _id: 'lim2', + value: 2 + }, + { + _id: 'lim3', + value: 3 + } + ]); + }); + + it('should reject non-integer limit values', async function() { + try { + await db.collection('sectest').find({}).limit('1; DROP TABLE users;--').toArray(); + expect.fail('Should have rejected non-integer limit'); + } catch (e) { + expect(e.message).to.include('must be a non-negative integer'); + } + }); + + it('should reject negative limit values', async function() { + try { + await db.collection('sectest').find({}).limit(-1).toArray(); + expect.fail('Should have rejected negative limit'); + } catch (e) { + expect(e.message).to.include('must be a non-negative integer'); + } + }); + + it('should reject non-integer skip values', async function() { + try { + await db.collection('sectest').find({}).skip('1; DROP TABLE users;--').toArray(); + expect.fail('Should have rejected non-integer skip'); + } catch (e) { + expect(e.message).to.include('must be a non-negative integer'); + } + }); + + it('should allow valid integer limit and skip', async function() { + const docs = await db.collection('sectest').find({}).skip(1).limit(1).toArray(); + expect(docs).to.have.lengthOf(1); + }); + }); + }); + } + + // Basic tests that apply to both adapters + describe('Basic Security', function() { + it('should prevent duplicate _id atomically', async function() { + await db.collection('sectest').insertOne({ + _id: 'atomic1', + value: 1 + }); + + const promises = []; + for (let i = 0; i < 5; i++) { + promises.push( + db.collection('sectest').insertOne({ + _id: 'atomic1', + value: i + }) + .then(() => 'success') + .catch(() => 'duplicate') + ); + } + + const results = await Promise.all(promises); + const successes = results.filter(r => r === 'success'); + expect(successes).to.have.lengthOf(0); // First insert already succeeded above + + // Only one document should exist + const count = await db.collection('sectest').countDocuments({ _id: 'atomic1' }); + expect(count).to.equal(1); + }); + }); +}); diff --git a/packages/db-connect/test/shared.test.js b/packages/db-connect/test/shared.test.js new file mode 100644 index 0000000000..e0fd78e962 --- /dev/null +++ b/packages/db-connect/test/shared.test.js @@ -0,0 +1,121 @@ +/* global describe, it */ +/* eslint-disable no-unused-expressions */ +const { expect } = require('chai'); +const { + extractAnchoredLiteralPrefix, + prefixUpperBound +} = require('../lib/shared'); + +describe('shared: extractAnchoredLiteralPrefix', function() { + it('extracts a plain anchored literal', function() { + const r = extractAnchoredLiteralPrefix(/^hello/); + expect(r).to.deep.equal({ + prefix: 'hello', + anchored: true + }); + }); + + it('handles escaped slashes and dots (the ApostropheCMS page path pattern)', function() { + const r = extractAnchoredLiteralPrefix(/^\/parent\/child\/./); + expect(r.prefix).to.equal('/parent/child/'); + expect(r.anchored).to.equal(true); + }); + + it('stops at the first unescaped metacharacter', function() { + expect(extractAnchoredLiteralPrefix(/^foo.*bar/).prefix).to.equal('foo'); + expect(extractAnchoredLiteralPrefix(/^foo(bar|baz)/).prefix).to.equal('foo'); + expect(extractAnchoredLiteralPrefix(/^foo\d+/).prefix).to.equal('foo'); + expect(extractAnchoredLiteralPrefix(/^foo[abc]/).prefix).to.equal('foo'); + expect(extractAnchoredLiteralPrefix(/^foo?/).prefix).to.equal('fo'); + }); + + it('keeps the preceding char before + (one-or-more, guaranteed once)', function() { + expect(extractAnchoredLiteralPrefix(/^foo+/).prefix).to.equal('foo'); + }); + + it('drops the preceding char before * (zero-or-more)', function() { + expect(extractAnchoredLiteralPrefix(/^foo*/).prefix).to.equal('fo'); + }); + + it('drops the preceding char before {0,...} quantifier', function() { + expect(extractAnchoredLiteralPrefix(/^foo{2,3}/).prefix).to.equal('fo'); + }); + + it('treats escaped metacharacters as literals', function() { + expect(extractAnchoredLiteralPrefix(/^a\.b\+c\*d/).prefix).to.equal('a.b+c*d'); + expect(extractAnchoredLiteralPrefix(/^a\(b\)c/).prefix).to.equal('a(b)c'); + }); + + it('returns empty prefix when not anchored', function() { + expect(extractAnchoredLiteralPrefix(/hello/)).to.deep.equal({ + prefix: '', + anchored: false + }); + }); + + it('returns empty prefix for case-insensitive regex', function() { + expect(extractAnchoredLiteralPrefix(/^hello/i)).to.deep.equal({ + prefix: '', + anchored: false + }); + }); + + it('returns empty prefix when the regex starts with a metacharacter', function() { + expect(extractAnchoredLiteralPrefix(/^.foo/).prefix).to.equal(''); + expect(extractAnchoredLiteralPrefix(/^(foo|bar)/).prefix).to.equal(''); + }); + + it('returns empty prefix for non-RegExp input', function() { + expect(extractAnchoredLiteralPrefix('hello').prefix).to.equal(''); + expect(extractAnchoredLiteralPrefix(null).prefix).to.equal(''); + }); + + it('stops at character-class escapes like \\d, \\w, \\s', function() { + expect(extractAnchoredLiteralPrefix(/^abc\d/).prefix).to.equal('abc'); + expect(extractAnchoredLiteralPrefix(/^abc\w/).prefix).to.equal('abc'); + expect(extractAnchoredLiteralPrefix(/^abc\s/).prefix).to.equal('abc'); + }); + + it('returns empty prefix for top-level alternation', function() { + // Top-level alternation means the other branch may match a completely + // different string, so no prefix is safe for range predicates. + expect(extractAnchoredLiteralPrefix(/^en:|^fr:/).prefix).to.equal(''); + expect(extractAnchoredLiteralPrefix(/^foo|^bar|^baz/).prefix).to.equal(''); + expect(extractAnchoredLiteralPrefix(/^abc|xyz/).prefix).to.equal(''); + }); + + it('returns empty prefix for just ^', function() { + expect(extractAnchoredLiteralPrefix(/^/).prefix).to.equal(''); + }); +}); + +describe('shared: prefixUpperBound', function() { + it('increments the last character code point', function() { + expect(prefixUpperBound('foo')).to.equal('fop'); + expect(prefixUpperBound('/parent/')).to.equal('/parent0'); // '/' (0x2F) -> '0' (0x30) + expect(prefixUpperBound('a')).to.equal('b'); + }); + + it('orders such that all P-prefixed strings fall in [P, upper)', function() { + const P = '/parent/'; + const U = prefixUpperBound(P); + // A few descendants + for (const s of [ '/parent/', '/parent/a', '/parent/child/foo', '/parent/~~~', '/parent/' + '\uFFFE' ]) { + expect(s >= P).to.equal(true); + expect(s < U).to.equal(true); + } + // Non-descendants must fall outside + for (const s of [ '/parent', '/parentx', '/parenu', '0', '/' ]) { + const inRange = s >= P && s < U; + expect(inRange).to.equal(false); + } + }); + + it('returns null for empty prefix', function() { + expect(prefixUpperBound('')).to.equal(null); + }); + + it('returns null when the last character is the max BMP code point', function() { + expect(prefixUpperBound('foo\uFFFF')).to.equal(null); + }); +}); diff --git a/packages/emulate-mongo-3-driver/package.json b/packages/emulate-mongo-3-driver/package.json index e367316a30..f9ce250fc9 100644 --- a/packages/emulate-mongo-3-driver/package.json +++ b/packages/emulate-mongo-3-driver/package.json @@ -28,6 +28,9 @@ "mongodb": "^6.8.0", "mongodb-legacy": "^6.0.1" }, + "apostropheTestConfig": { + "mongodbOnly": true + }, "devDependencies": { "eslint": "^9.39.1", "eslint-config-apostrophe": "workspace:^", diff --git a/packages/express-cache-on-demand/package.json b/packages/express-cache-on-demand/package.json index 9e845fd2d9..3810a1bae9 100644 --- a/packages/express-cache-on-demand/package.json +++ b/packages/express-cache-on-demand/package.json @@ -30,6 +30,9 @@ "cache-on-demand": "workspace:^", "lodash": "^4.18.1" }, + "apostropheTestConfig": { + "requiresMongo": false + }, "devDependencies": { "eslint": "^9.39.1", "eslint-config-apostrophe": "workspace:^", diff --git a/packages/form/test/test.js b/packages/form/test/test.js index 2be39e3193..7fc619df35 100644 --- a/packages/form/test/test.js +++ b/packages/form/test/test.js @@ -746,10 +746,12 @@ describe('Forms module', function () { it('should accept multiple files for a single file field when allowMultiple is true', async function () { // Update the existing form's file field to allow multiple + const formDoc = await apos.doc.db.findOne({ _id: savedForm1._id }); + const item = formDoc.contents.items.find(i => i._id === 'dogPhotoId'); + item.allowMultiple = true; await apos.doc.db.updateOne( { _id: savedForm1._id }, - { $set: { 'contents.items.$[w].allowMultiple': true } }, - { arrayFilters: [ { 'w._id': 'dogPhotoId' } ] } + { $set: { contents: formDoc.contents } } ); const formData = new FormData(); diff --git a/packages/import-export/test/importDraftsOnly.js b/packages/import-export/test/importDraftsOnly.js index 9248eed1ec..5ee09336da 100644 --- a/packages/import-export/test/importDraftsOnly.js +++ b/packages/import-export/test/importDraftsOnly.js @@ -69,6 +69,7 @@ describe('#import - when `importDraftsOnly` option is set to `true`', function ( const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = topics; @@ -127,6 +128,7 @@ describe('#import - when `importDraftsOnly` option is set to `true`', function ( const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = topics.map(topic => ({ @@ -169,6 +171,7 @@ describe('#import - when `importDraftsOnly` option is set to `true`', function ( const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = topics.map(topic => ({ @@ -220,6 +223,7 @@ describe('#import - when `importDraftsOnly` option is set to `true`', function ( const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = topics.map(topic => ({ @@ -263,6 +267,7 @@ describe('#import - when `importDraftsOnly` option is set to `true`', function ( const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = topics.map(topic => ({ @@ -305,6 +310,7 @@ describe('#import - when `importDraftsOnly` option is set to `true`', function ( const pages = await apos.doc.db .find({ type: 'default-page' }) + .sort({ _id: 1 }) .toArray(); const actual = pages.map(page => ({ @@ -384,6 +390,7 @@ describe('#import - when `importDraftsOnly` option is set to `true`', function ( const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = topics; @@ -466,6 +473,7 @@ describe('#import - when `importDraftsOnly` option is set to `true`', function ( const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = topics; @@ -528,6 +536,7 @@ describe('#import - when `importDraftsOnly` option is set to `true`', function ( const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = topics; @@ -589,6 +598,7 @@ describe('#import - when `importDraftsOnly` option is set to `true`', function ( const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = topics; @@ -644,6 +654,7 @@ describe('#import - when `importDraftsOnly` option is set to `true`', function ( const pages = await apos.doc.db .find({ type: 'default-page' }) + .sort({ _id: 1 }) .toArray(); const actual = pages; @@ -705,6 +716,7 @@ describe('#import - when `importDraftsOnly` option is set to `true`', function ( const pages = await apos.doc.db .find({ type: 'default-page' }) + .sort({ _id: 1 }) .toArray(); const actual = pages; diff --git a/packages/import-export/test/index.js b/packages/import-export/test/index.js index c3f53a1da4..d4fe8bd213 100644 --- a/packages/import-export/test/index.js +++ b/packages/import-export/test/index.js @@ -409,10 +409,12 @@ describe('@apostrophecms/import-export', function() { const importedDocs = await apos.doc.db .find({ type: /article|topic|@apostrophecms\/image/ }) + .sort({ _id: 1 }) .toArray(); const importedAttachments = await apos.attachment.db .find() + .sort({ _id: 1 }) .toArray(); const articlesWithRelatedImages = importedDocs @@ -428,7 +430,7 @@ describe('@apostrophecms/import-export', function() { const actual = { articlesWithRelatedImages, docsLength: importedDocs.length, - docsTitles: importedDocs.map(({ title }) => title), + docsTitles: importedDocs.map(({ title }) => title).sort(), attachmentsNames: importedAttachments.map(({ name }) => name), attachmentFileNames: attachmentFiles.map((fullName) => { const regex = /-([\w\d-]+)\./; @@ -451,10 +453,11 @@ describe('@apostrophecms/import-export', function() { } ], docsLength: 10, docsTitles: [ - 'article2', 'article1', - 'article2', 'article1', - 'topic1', 'topic3', 'topic2', - 'topic3', 'topic1', 'topic2' + 'article1', 'article1', + 'article2', 'article2', + 'topic1', 'topic1', + 'topic2', 'topic2', + 'topic3', 'topic3' ], attachmentsNames: [ 'test-image' ], attachmentFileNames: new Array(apos.attachment.imageSizes.length + 1) @@ -535,13 +538,14 @@ describe('@apostrophecms/import-export', function() { type: /article|topic|@apostrophecms\/image/, aposMode: { $ne: 'previous' } }) + .sort({ _id: 1 }) .toArray(); - const updatedAttachments = await apos.attachment.db.find().toArray(); + const updatedAttachments = await apos.attachment.db.find().sort({ _id: 1 }).toArray(); const attachmentFiles = await fs.readdir(attachmentPath); const job = await apos.modules['@apostrophecms/job'].db.findOne({ _id: jobId }); const actual = { - docTitles: updatedDocs.map(({ title }) => title), + docTitles: updatedDocs.map(({ title }) => title).sort(), attachmentNames: updatedAttachments.map(({ name }) => name), attachmentFileNames: extractFileNames(attachmentFiles), job: { @@ -552,18 +556,12 @@ describe('@apostrophecms/import-export', function() { const expected = { docTitles: [ - 'image1', - 'image1', - 'article1', - 'article2', - 'article1', - 'article2', - 'new title', - 'topic2', - 'topic1', - 'new title', - 'topic2', - 'topic1' + 'article1', 'article1', + 'article2', 'article2', + 'image1', 'image1', + 'new title', 'new title', + 'topic1', 'topic1', + 'topic2', 'topic2' ], attachmentNames: [ 'new-name' ], attachmentFileNames: new Array(apos.attachment.imageSizes.length + 1) @@ -609,20 +607,21 @@ describe('@apostrophecms/import-export', function() { const importedDocs = await apos.doc.db .find({ type: /default-page|article|topic|@apostrophecms\/image/ }) + .sort({ _id: 1 }) .toArray(); const importedAttachments = await apos.attachment.db.find( { aposMode: { $ne: 'previous' } } - ).toArray(); + ).sort({ _id: 1 }).toArray(); const attachmentFiles = await fs.readdir(attachmentPath); const actual = { - docTitles: importedDocs.map(({ title }) => title), + docTitles: importedDocs.map(({ title }) => title).sort(), attachmentNames: importedAttachments.map(({ name }) => name), attachmentFileNames: extractFileNames(attachmentFiles) }; const expected = { - docTitles: [ 'image1', 'image1', 'article2', 'article2', 'page1', 'page1' ], + docTitles: [ 'article2', 'article2', 'image1', 'image1', 'page1', 'page1' ], attachmentNames: [ 'test-image' ], attachmentFileNames: new Array(apos.attachment.imageSizes.length + 1) .fill('test-image') @@ -699,13 +698,14 @@ describe('@apostrophecms/import-export', function() { type: /default-page|article|@apostrophecms\/image/, aposMode: { $ne: 'previous' } }) + .sort({ _id: 1 }) .toArray(); - const updatedAttachments = await apos.attachment.db.find().toArray(); + const updatedAttachments = await apos.attachment.db.find().sort({ _id: 1 }).toArray(); const attachmentFiles = await fs.readdir(attachmentPath); const job = await apos.modules['@apostrophecms/job'].db.findOne({ _id: jobId }); const actual = { - docTitles: updatedDocs.map(({ title }) => title), + docTitles: updatedDocs.map(({ title }) => title).sort(), attachmentNames: updatedAttachments.map(({ name }) => name), attachmentFileNames: extractFileNames(attachmentFiles), job: { @@ -716,14 +716,10 @@ describe('@apostrophecms/import-export', function() { const expected = { docTitles: [ - 'image1', - 'image1', - 'new title', - 'article2', - 'new title', - 'article2', - 'page1', - 'page1' + 'article2', 'article2', + 'image1', 'image1', + 'new title', 'new title', + 'page1', 'page1' ], attachmentNames: [ 'new-name' ], attachmentFileNames: new Array(apos.attachment.imageSizes.length + 1) @@ -807,13 +803,14 @@ describe('@apostrophecms/import-export', function() { type: /default-page|article|@apostrophecms\/image/, aposMode: { $ne: 'previous' } }) + .sort({ _id: 1 }) .toArray(); - const updatedAttachments = await apos.attachment.db.find().toArray(); + const updatedAttachments = await apos.attachment.db.find().sort({ _id: 1 }).toArray(); const attachmentFiles = await fs.readdir(attachmentPath); const job = await apos.modules['@apostrophecms/job'].db.findOne({ _id: jobId }); const actual = { - docTitles: docs.map(({ title }) => title), + docTitles: docs.map(({ title }) => title).sort(), attachmentNames: updatedAttachments.map(({ name }) => name), attachmentFileNames: extractFileNames(attachmentFiles), job: { @@ -825,14 +822,10 @@ describe('@apostrophecms/import-export', function() { const expected = { docTitles: [ - 'new title', - 'new title', - 'new title', - 'article2', - 'new title', - 'article2', - 'page1', - 'page1' + 'article2', 'article2', + 'new title', 'new title', + 'new title', 'new title', + 'page1', 'page1' ], attachmentNames: [ 'new-name' ], attachmentFileNames: new Array(apos.attachment.imageSizes.length + 1) @@ -918,6 +911,7 @@ describe('@apostrophecms/import-export', function() { const updatedPage = await apos.doc.db .find({ title: 'page2' }) + .sort({ _id: 1 }) .toArray(); assert.deepEqual(updatedPage.every((doc) => { @@ -1091,6 +1085,7 @@ describe('@apostrophecms/import-export', function() { const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); assert.equal(topics.length, 1); @@ -1107,6 +1102,7 @@ describe('@apostrophecms/import-export', function() { const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); assert.equal(topics.length, 1); @@ -1122,6 +1118,7 @@ describe('@apostrophecms/import-export', function() { const pages = await apos.doc.db .find({ type: 'default-page' }) + .sort({ _id: 1 }) .toArray(); assert.equal(pages.length, 1); @@ -1136,6 +1133,7 @@ describe('@apostrophecms/import-export', function() { const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); assert.equal(topics.length, 2); @@ -1158,6 +1156,7 @@ describe('@apostrophecms/import-export', function() { const pages = await apos.doc.db .find({ type: 'default-page' }) + .sort({ _id: 1 }) .toArray(); assert.equal(pages.length, 2); @@ -1178,6 +1177,7 @@ describe('@apostrophecms/import-export', function() { const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); assert.equal(topics.length, 2); @@ -1200,6 +1200,7 @@ describe('@apostrophecms/import-export', function() { const pages = await apos.doc.db .find({ type: 'default-page' }) + .sort({ _id: 1 }) .toArray(); assert.equal(pages.length, 2); @@ -1227,6 +1228,7 @@ describe('@apostrophecms/import-export', function() { const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); assert.equal(topics.length, 2); @@ -1258,6 +1260,7 @@ describe('@apostrophecms/import-export', function() { const pages = await apos.doc.db .find({ type: 'default-page' }) + .sort({ _id: 1 }) .toArray(); assert.equal(pages.length, 2); @@ -1301,6 +1304,7 @@ describe('@apostrophecms/import-export', function() { const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); assert.equal(topics.length, 2); @@ -1345,6 +1349,7 @@ describe('@apostrophecms/import-export', function() { const pages = await apos.doc.db .find({ type: 'default-page' }) + .sort({ _id: 1 }) .toArray(); assert.equal(pages.length, 2); @@ -1388,6 +1393,7 @@ describe('@apostrophecms/import-export', function() { const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); assert.equal(topics.length, 2); @@ -1432,6 +1438,7 @@ describe('@apostrophecms/import-export', function() { const pages = await apos.doc.db .find({ type: 'default-page' }) + .sort({ _id: 1 }) .toArray(); assert.equal(pages.length, 2); @@ -1709,6 +1716,7 @@ describe('@apostrophecms/import-export', function() { ] } }) + .sort({ _id: 1 }) .toArray(); const actual = importedDocs.map(doc => { @@ -2029,6 +2037,7 @@ describe('@apostrophecms/import-export', function() { ] } }) + .sort({ _id: 1 }) .toArray(); const actual = importedDocs.map(doc => { diff --git a/packages/import-export/test/overrideDuplicates.js b/packages/import-export/test/overrideDuplicates.js index d813cedda4..e7785e49b6 100644 --- a/packages/import-export/test/overrideDuplicates.js +++ b/packages/import-export/test/overrideDuplicates.js @@ -277,7 +277,7 @@ describe('#overrideDuplicates - overriding locales integration tests', function( locale: 'fr', mode: 'draft' }); - const [ nonLocalized ] = await apos.doc.db.find({ title: 'nonLocalized1' }).toArray(); + const [ nonLocalized ] = await apos.doc.db.find({ title: 'nonLocalized1' }).sort({ _id: 1 }).toArray(); const enArticles = await apos.article.find(req).toArray(); const parkedPages = await apos.page .find(req, { parkedId: { $exists: true } }) diff --git a/packages/import-export/test/overrideLocales.js b/packages/import-export/test/overrideLocales.js index ad89a7e054..d60547dcd5 100644 --- a/packages/import-export/test/overrideLocales.js +++ b/packages/import-export/test/overrideLocales.js @@ -68,6 +68,7 @@ describe('#import - overriding locales integration tests', function() { .findOne({ _id: notificationId }); const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = { @@ -123,6 +124,7 @@ describe('#import - overriding locales integration tests', function() { .findOne({ _id: notificationId }); const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = { @@ -267,6 +269,7 @@ describe('#import - overriding locales integration tests', function() { .findOne({ _id: notificationId }); const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = { @@ -327,6 +330,7 @@ describe('#import - overriding locales integration tests', function() { .findOne({ _id: notificationId }); const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = { @@ -390,6 +394,7 @@ describe('#import - overriding locales integration tests', function() { .findOne({ _id: notificationId2 }); const topics = await apos.doc.db .find({ type: 'topic' }) + .sort({ _id: 1 }) .toArray(); const actual = { diff --git a/packages/mongodb-snapshot/package.json b/packages/mongodb-snapshot/package.json index b8617f32dd..7a0f058c21 100644 --- a/packages/mongodb-snapshot/package.json +++ b/packages/mongodb-snapshot/package.json @@ -22,6 +22,9 @@ "bson": "^6.10.1", "mongodb": "^6.12.0" }, + "apostropheTestConfig": { + "mongodbOnly": true + }, "devDependencies": { "mocha": "^11.7.5" }, diff --git a/packages/uploadfs/webp-test.js b/packages/uploadfs/webp-test.js old mode 100755 new mode 100644 From dfd25cf35c1371801971b323875ace51098c250e Mon Sep 17 00:00:00 2001 From: Tom Boutell Date: Fri, 24 Apr 2026 09:31:23 -0400 Subject: [PATCH 08/56] forgot to include a changeset (#5390) --- .changeset/curvy-bobcats-peel.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/curvy-bobcats-peel.md diff --git a/.changeset/curvy-bobcats-peel.md b/.changeset/curvy-bobcats-peel.md new file mode 100644 index 0000000000..0849faf743 --- /dev/null +++ b/.changeset/curvy-bobcats-peel.md @@ -0,0 +1,5 @@ +--- +"apostrophe": minor +--- + +Introduced support for postgres://, sqlite://, and multipostgres:// database URIs in addition to mongodb://. The new db-connect API supports all of the database operations currently used in our own core, pro and multisite modules. For more information see the documentation. From b360b05e8d4e479a36fcae45a5a6dbf820b8fba3 Mon Sep 17 00:00:00 2001 From: Stuart Romanek Date: Wed, 29 Apr 2026 14:00:17 -0400 Subject: [PATCH 09/56] ignore inline table array as draggable ui for windows (#5392) --- .changeset/stale-shirts-read.md | 5 +++++ .../@apostrophecms/modal/ui/apos/components/AposModal.vue | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changeset/stale-shirts-read.md diff --git a/.changeset/stale-shirts-read.md b/.changeset/stale-shirts-read.md new file mode 100644 index 0000000000..271cd86872 --- /dev/null +++ b/.changeset/stale-shirts-read.md @@ -0,0 +1,5 @@ +--- +"apostrophe": patch +--- + +fixes issue where orderable table array items drag the entire floating window diff --git a/packages/apostrophe/modules/@apostrophecms/modal/ui/apos/components/AposModal.vue b/packages/apostrophe/modules/@apostrophecms/modal/ui/apos/components/AposModal.vue index 6402d79c49..a83770ff35 100644 --- a/packages/apostrophe/modules/@apostrophecms/modal/ui/apos/components/AposModal.vue +++ b/packages/apostrophe/modules/@apostrophecms/modal/ui/apos/components/AposModal.vue @@ -212,7 +212,8 @@ const nonDraggableElements = [ '.apos-field--inline-array-field', '.apos-field--inline-array-table-with-remove-button-field', '.apos-field--inline-array-table-field', - '.apos-input-color__sample-picker' + '.apos-input-color__sample-picker', + '.apos-input-array-inline-table' ]; const resizeSides = [ From 77a2968206b0208883ff45de37feb43ba0b5d5b1 Mon Sep 17 00:00:00 2001 From: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Date: Mon, 4 May 2026 13:26:48 +0300 Subject: [PATCH 10/56] Layout focus orchestration (#5393) --- .changeset/quick-guests-join.md | 5 +++++ .../area/ui/apos/components/AposAreaWidget.vue | 2 ++ .../ui/apos/components/AposAreaLayoutEditor.vue | 6 ++++++ 3 files changed, 13 insertions(+) create mode 100644 .changeset/quick-guests-join.md diff --git a/.changeset/quick-guests-join.md b/.changeset/quick-guests-join.md new file mode 100644 index 0000000000..8bab7b0384 --- /dev/null +++ b/.changeset/quick-guests-join.md @@ -0,0 +1,5 @@ +--- +"apostrophe": minor +--- + +Fix layout widget not regaining full focus on switching back to Edit content mode. diff --git a/packages/apostrophe/modules/@apostrophecms/area/ui/apos/components/AposAreaWidget.vue b/packages/apostrophe/modules/@apostrophecms/area/ui/apos/components/AposAreaWidget.vue index a8ac47dde0..7b83852860 100644 --- a/packages/apostrophe/modules/@apostrophecms/area/ui/apos/components/AposAreaWidget.vue +++ b/packages/apostrophe/modules/@apostrophecms/area/ui/apos/components/AposAreaWidget.vue @@ -536,6 +536,7 @@ export default { apos.bus.$on('widget-focus-parent', this.focusParent); apos.bus.$on('context-menu-toggled', this.getFocusForMenu); apos.bus.$on('suppress-focused-widget-controls', this.doSuppressWidgetControls); + apos.bus.$on('clear-focused-widget-control-suppression', this.clearSuppressionFlags); this.breadcrumbs.$lastEl = this.$el; @@ -573,6 +574,7 @@ export default { // Remove the focus parent listener when unmounted apos.bus.$off('widget-focus-parent', this.focusParent); apos.bus.$off('suppress-focused-widget-controls', this.doSuppressWidgetControls); + apos.bus.$off('clear-focused-widget-control-suppression', this.clearSuppressionFlags); window.removeEventListener('scroll', this.stickyControlsScroll); window.removeEventListener('resize', this.stickyControlsResize); this.unregisterFromGraph(); diff --git a/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposAreaLayoutEditor.vue b/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposAreaLayoutEditor.vue index 17a32f0202..9f4f2387d4 100644 --- a/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposAreaLayoutEditor.vue +++ b/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposAreaLayoutEditor.vue @@ -259,6 +259,12 @@ export default { this.emphasizeGrid(); } else { this.deEmphasizeGrid(); + // Leaving layout mode: clear the control-suppression flag that was + // set on the focused widget when we entered layout mode. Without + // this the layout widget keeps `isSuppressingWidgetControls=true` + // and its side operation bar stays hidden even though it is the + // focused widget. + apos.bus.$emit('clear-focused-widget-control-suppression'); } } }, From 82ff1104c635d187420f29a6dfef748e7a8609ab Mon Sep 17 00:00:00 2001 From: Robert Means Date: Mon, 4 May 2026 11:34:44 -0400 Subject: [PATCH 11/56] Pro 9405 remove hreflang (#5395) * Remove hreflang generation and update README * Add changeset * Changeset update --- .changeset/common-beans-lie.md | 5 +++ packages/seo/README.md | 2 ++ packages/seo/lib/nodes.js | 63 +--------------------------------- 3 files changed, 8 insertions(+), 62 deletions(-) create mode 100644 .changeset/common-beans-lie.md diff --git a/.changeset/common-beans-lie.md b/.changeset/common-beans-lie.md new file mode 100644 index 0000000000..42fd0945d7 --- /dev/null +++ b/.changeset/common-beans-lie.md @@ -0,0 +1,5 @@ +--- +"@apostrophecms/seo": minor +--- + +Removes unimplemented hreflang output; use @apostrophecms/sitemap for hreflang support diff --git a/packages/seo/README.md b/packages/seo/README.md index 17dd7f5a27..ecc080959d 100644 --- a/packages/seo/README.md +++ b/packages/seo/README.md @@ -297,6 +297,8 @@ The module automatically provides an `/llms.txt` route to communicate your AI us Works seamlessly with `@apostrophecms/sitemap` to generate XML sitemaps that help search engines discover and index your content. The sitemap is automatically referenced in the `/llms.txt` file for AI crawlers. +**Hreflang:** This module does not output `` tags. Generating them at render time requires fetching each locale's true URL, which is expensive. Google accepts hreflang via XML sitemap as an equivalent signal, and [`@apostrophecms/sitemap`](https://apostrophecms.com/extensions/sitemap) handles this without the per-request overhead. + ### Performance Optimization **Critical Font Preloading** - Automatically preload critical fonts to improve Core Web Vitals scores and SEO performance: diff --git a/packages/seo/lib/nodes.js b/packages/seo/lib/nodes.js index 5a9c8461c6..01731cfcae 100644 --- a/packages/seo/lib/nodes.js +++ b/packages/seo/lib/nodes.js @@ -262,66 +262,6 @@ function getMetaHead(data, options) { } } - // Add hreflang tags - const hreflangTags = getHreflangTags(data); - nodes.push(...hreflangTags); - - return nodes; -} - -function getHreflangTags(data) { - const nodes = []; - const { - piece, page, req - } = data; - const document = piece || page; - - // Only add hreflang if the site has i18n enabled - if (!req?.locale || !document?._url) { - return nodes; - } - - // Get all locale versions of this document if they exist - // This assumes ApostropheCMS i18n structure - if (document.aposLocale && data.alternateLocales) { - // Add current page - nodes.push({ - name: 'link', - attrs: { - rel: 'alternate', - hreflang: req.locale, - href: document._url - } - }); - - // Add alternate locales - data.alternateLocales.forEach(alt => { - if (alt.locale && alt.url) { - nodes.push({ - name: 'link', - attrs: { - rel: 'alternate', - hreflang: alt.locale, - href: alt.url - } - }); - } - }); - - // Add x-default for the primary locale - const defaultLocale = data.alternateLocales.find(alt => alt.isDefault); - if (defaultLocale) { - nodes.push({ - name: 'link', - attrs: { - rel: 'alternate', - hreflang: 'x-default', - href: defaultLocale.url - } - }); - } - } - return nodes; } @@ -407,6 +347,5 @@ function getTagManagerBody(data) { module.exports = { getMetaHead, getTagManagerHead, - getTagManagerBody, - getHreflangTags + getTagManagerBody }; From d65c5d61032f59b76189ecd18005436b2c3326d9 Mon Sep 17 00:00:00 2001 From: Robert Means Date: Mon, 4 May 2026 11:36:22 -0400 Subject: [PATCH 12/56] Pro 9406 base url (#5396) * Removes `seoSiteCanonicalUrl` * Update tests and remove missed log * Change semver level --- .changeset/shaky-regions-spend.md | 5 +++++ packages/seo/i18n/de.json | 2 -- packages/seo/i18n/en.json | 2 -- packages/seo/i18n/es.json | 2 -- packages/seo/i18n/fr.json | 2 -- packages/seo/i18n/it.json | 2 -- packages/seo/i18n/pt-BR.json | 2 -- packages/seo/i18n/sk.json | 2 -- packages/seo/lib/jsonld-schemas.js | 8 -------- .../@apostrophecms/seo-fields-global/index.js | 20 +++++++++++-------- packages/seo/test/functional-tests.js | 10 +++------- packages/seo/test/unit-tests.js | 2 -- 12 files changed, 20 insertions(+), 39 deletions(-) create mode 100644 .changeset/shaky-regions-spend.md diff --git a/.changeset/shaky-regions-spend.md b/.changeset/shaky-regions-spend.md new file mode 100644 index 0000000000..a21e913af0 --- /dev/null +++ b/.changeset/shaky-regions-spend.md @@ -0,0 +1,5 @@ +--- +"@apostrophecms/seo": minor +--- + +Removes the `seoSiteCanonicalUrl` field from global settings. The base URL is now derived automatically from `APOS_BASE_URL` or the `baseUrl` option. The value remains available at `req.data.global.seoSiteCanonicalUrl` for backwards compatibility. diff --git a/packages/seo/i18n/de.json b/packages/seo/i18n/de.json index aa74b5dd98..1a1c35e8e0 100644 --- a/packages/seo/i18n/de.json +++ b/packages/seo/i18n/de.json @@ -30,8 +30,6 @@ "siteNameHelp": "Der Name Ihrer Website (verwendet in strukturierten Daten und beim Teilen in sozialen Netzwerken).", "siteDescription": "Website-Beschreibung", "siteDescriptionHelp": "Eine kurze Beschreibung Ihrer Website.", - "siteCanonicalUrl": "Basis-URL der Website", - "siteCanonicalUrlHelp": "Die Basis-URL Ihrer Website (z. B. https://example.com). Erforderlich für korrekte kanonische Links und strukturierte Daten.", "themeColor": "Mobile Themenfarbe", "themeColorHelp": "Legen Sie die Farbe der Browser-Oberfläche für Mobilgeräte und PWAs fest.", diff --git a/packages/seo/i18n/en.json b/packages/seo/i18n/en.json index b5eaffcf59..d5205dbe89 100644 --- a/packages/seo/i18n/en.json +++ b/packages/seo/i18n/en.json @@ -30,8 +30,6 @@ "siteNameHelp": "The name of your website (used in structured data and social sharing).", "siteDescription": "Site Description", "siteDescriptionHelp": "A brief description of your website.", - "siteCanonicalUrl": "Site Base URL", - "siteCanonicalUrlHelp": "The base URL of your site (e.g., https://example.com). Required for proper canonical links and structured data.", "themeColor": "Mobile Theme Color", "themeColorHelp": "Set the browser chrome color for mobile devices and PWAs.", diff --git a/packages/seo/i18n/es.json b/packages/seo/i18n/es.json index cd59918887..72ae861be9 100644 --- a/packages/seo/i18n/es.json +++ b/packages/seo/i18n/es.json @@ -30,8 +30,6 @@ "siteNameHelp": "El nombre de su sitio web (usado en datos estructurados y en redes sociales).", "siteDescription": "Descripción del sitio", "siteDescriptionHelp": "Una breve descripción de su sitio web.", - "siteCanonicalUrl": "URL base del sitio", - "siteCanonicalUrlHelp": "La URL base de su sitio (ej. https://example.com). Requerida para enlaces canónicos y datos estructurados correctos.", "themeColor": "Color del tema móvil", "themeColorHelp": "Establezca el color de la interfaz del navegador en dispositivos móviles y PWAs.", diff --git a/packages/seo/i18n/fr.json b/packages/seo/i18n/fr.json index 793ba8e7e5..84887ed702 100644 --- a/packages/seo/i18n/fr.json +++ b/packages/seo/i18n/fr.json @@ -30,8 +30,6 @@ "siteNameHelp": "Le nom de votre site web (utilisé dans les données structurées et le partage sur les réseaux sociaux).", "siteDescription": "Description du site", "siteDescriptionHelp": "Brève description de votre site web.", - "siteCanonicalUrl": "URL de base du site", - "siteCanonicalUrlHelp": "L’URL de base de votre site (ex. https://example.com). Nécessaire pour des liens canoniques et des données structurées corrects.", "themeColor": "Couleur du thème mobile", "themeColorHelp": "Définissez la couleur de l’interface du navigateur sur les appareils mobiles et les PWA.", diff --git a/packages/seo/i18n/it.json b/packages/seo/i18n/it.json index e4f66b05ea..042871d5f3 100644 --- a/packages/seo/i18n/it.json +++ b/packages/seo/i18n/it.json @@ -30,8 +30,6 @@ "siteNameHelp": "Il nome del tuo sito web (usato nei dati strutturati e nella condivisione sui social).", "siteDescription": "Descrizione del sito", "siteDescriptionHelp": "Breve descrizione del tuo sito web.", - "siteCanonicalUrl": "URL base del sito", - "siteCanonicalUrlHelp": "L’URL base del tuo sito (es. https://example.com). Necessario per i link canonici e i dati strutturati corretti.", "themeColor": "Colore tema mobile", "themeColorHelp": "Imposta il colore dell’interfaccia del browser per dispositivi mobili e PWA.", diff --git a/packages/seo/i18n/pt-BR.json b/packages/seo/i18n/pt-BR.json index 2de412cf89..0e2b96321f 100644 --- a/packages/seo/i18n/pt-BR.json +++ b/packages/seo/i18n/pt-BR.json @@ -29,8 +29,6 @@ "siteNameHelp": "O nome do seu site (usado em dados estruturados e compartilhamento em redes sociais).", "siteDescription": "Descrição do site", "siteDescriptionHelp": "Uma breve descrição do seu site.", - "siteCanonicalUrl": "URL base do site", - "siteCanonicalUrlHelp": "A URL base do seu site (ex.: https://example.com). Necessária para links canônicos e dados estruturados corretos.", "themeColor": "Cor do tema móvel", "themeColorHelp": "Defina a cor da interface do navegador para dispositivos móveis e PWAs.", diff --git a/packages/seo/i18n/sk.json b/packages/seo/i18n/sk.json index 9794e9ad7b..c639e2b234 100644 --- a/packages/seo/i18n/sk.json +++ b/packages/seo/i18n/sk.json @@ -30,8 +30,6 @@ "siteNameHelp": "Názov vašej webovej stránky (používa sa v štruktúrovaných dátach a pri zdieľaní na sociálnych sieťach).", "siteDescription": "Popis stránky", "siteDescriptionHelp": "Stručný popis vašej webovej stránky.", - "siteCanonicalUrl": "Základná URL stránky", - "siteCanonicalUrlHelp": "Základná URL vašej stránky (napr. https://example.com). Potrebné pre správne kanonické odkazy a štruktúrované dáta.", "themeColor": "Farba témy pre mobil", "themeColorHelp": "Nastavte farbu rozhrania prehliadača pre mobilné zariadenia a PWA.", diff --git a/packages/seo/lib/jsonld-schemas.js b/packages/seo/lib/jsonld-schemas.js index ae4c08d02b..2ea026cbda 100644 --- a/packages/seo/lib/jsonld-schemas.js +++ b/packages/seo/lib/jsonld-schemas.js @@ -983,14 +983,6 @@ class JsonLdSchemaHandler { if (fromGlobal) { return fromGlobal.replace(/\/$/, ''); } - const abs = data.req?.absoluteUrl; - - if (abs) { - try { - return new URL(abs).origin; - } catch (e) { } - } - return null; } diff --git a/packages/seo/modules/@apostrophecms/seo-fields-global/index.js b/packages/seo/modules/@apostrophecms/seo-fields-global/index.js index 93b2ada0ca..457793c60d 100644 --- a/packages/seo/modules/@apostrophecms/seo-fields-global/index.js +++ b/packages/seo/modules/@apostrophecms/seo-fields-global/index.js @@ -141,12 +141,6 @@ module.exports = { textarea: true, help: 'aposSeo:siteDescriptionHelp' }, - seoSiteCanonicalUrl: { - label: 'aposSeo:siteCanonicalUrl', - type: 'url', - required: true, - help: 'aposSeo:siteCanonicalUrlHelp' - }, seoJsonLdOrganization: { label: 'aposSeo:organizationInfo', type: 'object', @@ -333,7 +327,6 @@ module.exports = { 'llmsCustomText', 'seoSiteName', 'seoSiteDescription', - 'seoSiteCanonicalUrl', 'seoJsonLdOrganization', 'seoSocialProfiles', 'seoSearchQueryParam' @@ -375,6 +368,17 @@ module.exports = { : null; }, + extendMethods(self) { + return { + async addGlobalToData(_super, req) { + await _super(req); + // For bc. Prefer self.apos.url.getBaseUrl(req) or the ._url + // property of the document in question. + req.data.global.seoSiteCanonicalUrl = self.apos.url.getBaseUrl(req); + } + }; + }, + handlers(self) { return { '@apostrophecms/url:getAllUrlMetadata': { @@ -533,7 +537,7 @@ Allow: / return global.llmsCustomText; } - const baseUrl = global?.seoSiteCanonicalUrl || req.absoluteUrl.split('/').slice(0, 3).join('/'); + const baseUrl = self.apos.url.getBaseUrl(req); // Build the LLMS.txt content let content = `# ${global.seoSiteName || global.title || 'Website'}\n\n`; diff --git a/packages/seo/test/functional-tests.js b/packages/seo/test/functional-tests.js index 8c85f2a25b..e893f59fab 100644 --- a/packages/seo/test/functional-tests.js +++ b/packages/seo/test/functional-tests.js @@ -14,6 +14,7 @@ describe('@apostrophecms/seo - Integration Tests (Actual Page Output)', function apos = await t.create({ root: module, testModule: true, + baseUrl: 'http://localhost:3000', modules: getAppConfig() }); }); @@ -37,7 +38,6 @@ describe('@apostrophecms/seo - Integration Tests (Actual Page Output)', function { _id: global._id }, { $set: { - seoSiteCanonicalUrl: 'https://example.com', seoSiteName: 'Example Tech Corp', seoSiteDescription: 'Leading provider of tech solutions', seoJsonLdOrganization: { @@ -92,7 +92,7 @@ describe('@apostrophecms/seo - Integration Tests (Actual Page Output)', function const websiteSchema = jsonLd['@graph'].find(s => s['@type'] === 'WebSite'); assert(websiteSchema, 'Homepage should include WebSite schema'); assert.strictEqual(websiteSchema.name, 'Example Tech Corp'); - assert.strictEqual(websiteSchema.url, 'https://example.com'); + assert.strictEqual(websiteSchema.url, 'http://localhost:3000'); assert(websiteSchema.potentialAction, 'WebSite should have search action'); assert.strictEqual(websiteSchema.potentialAction['@type'], 'SearchAction'); @@ -158,7 +158,6 @@ describe('@apostrophecms/seo - Integration Tests (Actual Page Output)', function { _id: global._id }, { $set: { - seoSiteCanonicalUrl: 'https://example.com', seoSiteName: 'Test Site', seoJsonLdOrganization: { name: 'Test Organization', @@ -264,7 +263,6 @@ describe('@apostrophecms/seo - Integration Tests (Actual Page Output)', function { _id: global._id }, { $set: { - seoSiteCanonicalUrl: 'https://example.com', seoSiteName: 'Test Site' } } @@ -315,7 +313,6 @@ describe('@apostrophecms/seo - Integration Tests (Actual Page Output)', function { _id: global._id }, { $set: { - seoSiteCanonicalUrl: 'https://example.com', seoSiteName: 'Test Site' } } @@ -443,7 +440,6 @@ describe('@apostrophecms/seo - Integration Tests (Actual Page Output)', function { _id: global._id }, { $set: { - seoSiteCanonicalUrl: 'https://example.com', seoSiteName: 'Test Site for LLMs', seoSiteDescription: 'A test site demonstrating llms.txt', llmsTxtSelection: 'allow' @@ -460,7 +456,7 @@ describe('@apostrophecms/seo - Integration Tests (Actual Page Output)', function llmsTxt.includes('A test site demonstrating llms.txt'), 'Should include site description' ); - assert(llmsTxt.includes('https://example.com'), 'Should include base URL'); + assert(llmsTxt.includes('http://localhost:3000'), 'Should include base URL'); assert( llmsTxt.includes('## AI Training Policy') || llmsTxt.includes('AI Training'), diff --git a/packages/seo/test/unit-tests.js b/packages/seo/test/unit-tests.js index 241660ee63..095d82077f 100644 --- a/packages/seo/test/unit-tests.js +++ b/packages/seo/test/unit-tests.js @@ -78,8 +78,6 @@ describe('@apostrophecms/seo', function () { const schemas = handler.generateSchemas(data); const articleSchema = schemas.find(s => s['@type'] === 'Article'); - console.log(articleSchema);// eslint-disable-line no-console - assert(articleSchema.author, 'author should exist'); assert.strictEqual(articleSchema.author.name, 'John Doe'); }); From 008417fa9814dba4899b53b9394778dbd46ccf3c Mon Sep 17 00:00:00 2001 From: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Date: Tue, 5 May 2026 18:02:03 +0300 Subject: [PATCH 13/56] Bump dependencies (#5398) * Bump dependencies * Fix missing test await resulting in random failures --- packages/cli/lib/commands/create.js | 12 ++++++------ packages/cli/package.json | 2 +- packages/login-totp/test/test.js | 2 +- packages/oembetter/package.json | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/cli/lib/commands/create.js b/packages/cli/lib/commands/create.js index 9837cf9a1f..8596c3c8bb 100644 --- a/packages/cli/lib/commands/create.js +++ b/packages/cli/lib/commands/create.js @@ -86,7 +86,7 @@ module.exports = function (program) { const expressIndexPath = resolvePath('lib/modules/apostrophe-express/index.js'); if (fs.existsSync(expressIndexPath)) { util.replaceInFiles( - [expressIndexPath], + [ expressIndexPath ], /secret: undefined/, `secret: '${secret}'` ); @@ -96,7 +96,7 @@ module.exports = function (program) { secret = util.secret(); util.replaceInFiles( - [resolvePath('app.js')], + [ resolvePath('app.js') ], /disabledFileKey: undefined/, `disabledFileKey: '${secret}'` ); @@ -124,7 +124,7 @@ module.exports = function (program) { }); } catch (error) { await util.error('create', 'Error installing backend packages'); - /* eslint-disable-next-line no-console */ + console.error(error); } @@ -136,7 +136,7 @@ module.exports = function (program) { }); } catch (error) { await util.error('create', 'Error installing frontend packages'); - /* eslint-disable-next-line no-console */ + console.error(error); } } else { @@ -147,7 +147,7 @@ module.exports = function (program) { }); } catch (error) { await util.error('create', 'Error installing packages'); - /* eslint-disable-next-line no-console */ + console.error(error); } } @@ -192,6 +192,6 @@ module.exports = function (program) { }; function replaceInConfig(regex, replacement, resolvePath) { - const files = [resolvePath('app.js'), resolvePath('package.json')]; + const files = [ resolvePath('app.js'), resolvePath('package.json') ]; util.replaceInFiles(files, regex, replacement); } diff --git a/packages/cli/package.json b/packages/cli/package.json index ff9c72b249..20ff3c1395 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -29,7 +29,7 @@ "shell-quote": "^1.8.1", "shelljs": "^0.10.0", "sync-request": "^6.0.0", - "uuid": "^8.0.0" + "uuid": "^11.1.0" }, "devDependencies": { "eslint": "^9.39.1", diff --git a/packages/login-totp/test/test.js b/packages/login-totp/test/test.js index 0a298c2a9a..75e536c8e3 100644 --- a/packages/login-totp/test/test.js +++ b/packages/login-totp/test/test.js @@ -67,7 +67,7 @@ describe('totp module', function () { user.role = 'editor'; const doc = await apos.user.insert(apos.task.getReq(), user); - apos.user.safe.updateOne({ + await apos.user.safe.updateOne({ _id: doc._id }, { $set: { diff --git a/packages/oembetter/package.json b/packages/oembetter/package.json index 3b01690fec..63ee9c08c2 100644 --- a/packages/oembetter/package.json +++ b/packages/oembetter/package.json @@ -29,7 +29,7 @@ "dependencies": { "async": "^0.9.0", "cheerio": "^1.1.0", - "fast-xml-parser": "^4.5.6", + "fast-xml-parser": "^5.7.0", "node-fetch": "^2.6.7", "urls": "0.0.4" }, From 7ab9961a5f368f140e014a8099b4ca2f8fdfd284 Mon Sep 17 00:00:00 2001 From: Robert Means Date: Wed, 6 May 2026 08:01:32 -0400 Subject: [PATCH 14/56] Pro 9405 remove hreflang (#5395) * Remove hreflang generation and update README * Add changeset * Changeset update From e9b3bac692315414da47fe05202e8854dd31d7ee Mon Sep 17 00:00:00 2001 From: Tom Boutell Date: Wed, 6 May 2026 14:13:29 -0400 Subject: [PATCH 15/56] PRO-9467: remove defunct and nonpublic oembed providers and improve developer control to ensure security in the future (no risk exists today) (#5400) * PRO-9467: remove defunct and nonpublic oembed providers and improve developer control to ensure security in the future (no risk exists today) * allow newer twitter domain * infogr.am still around * facebook no longer does oembed --- .changeset/wild-forks-fetch.md | 15 ++++++++ .../modules/@apostrophecms/oembed/index.js | 31 +++++++++------- packages/oembetter/index.js | 36 +++++-------------- 3 files changed, 42 insertions(+), 40 deletions(-) create mode 100644 .changeset/wild-forks-fetch.md diff --git a/.changeset/wild-forks-fetch.md b/.changeset/wild-forks-fetch.md new file mode 100644 index 0000000000..f5f944c79e --- /dev/null +++ b/.changeset/wild-forks-fetch.md @@ -0,0 +1,15 @@ +--- +"apostrophe": minor +--- + +apostrophe and oembetter have been updated to eliminate a number of services that formerly supported +oembed for the general public, but no longer do so. While there is no security risk today, removing +these ensures that if these domains are ever allowed to lapse, they do not become an XSS +attack vector in the future. + +Because oembed responses are not always iframes, it is important that this list be maintained +over time. In addition, developers always have the option to prune it on their own by setting +the new minimumAllowlist and minimumEndpoints options of the @apostrophecms/oembed module. + +Thanks to [Sainithin0309](https://github.com/Sainithin0309) for pointing out the potential +long-term security concern. diff --git a/packages/apostrophe/modules/@apostrophecms/oembed/index.js b/packages/apostrophe/modules/@apostrophecms/oembed/index.js index 9e4e83ad8a..d21e458da7 100644 --- a/packages/apostrophe/modules/@apostrophecms/oembed/index.js +++ b/packages/apostrophe/modules/@apostrophecms/oembed/index.js @@ -16,10 +16,13 @@ const cheerio = require('cheerio'); // widely trusted sites are already allowlisted. // // Your `allowlist` option is concatenated with `oembetter`'s standard -// allowlist, plus wufoo.com, infogr.am, and slideshare.net. +// allowlist, plus wufoo.com, infogr.am and slideshare.net. // // Your `endpoints` option is concatenated with `oembetter`'s standard // endpoints list. +// +// If you wish to completely override the behavior, set +// `minimumAllowlist` and `minimumEndpoints` instead. module.exports = { options: { @@ -42,28 +45,30 @@ module.exports = { // Don't permit oembed of untrusted sites, which could // lead to XSS attacks - self.oembetter.allowlist(self.oembetter.suggestedAllowlist.concat( - self.options.allowlist || [], - [ - 'wufoo.com', - 'infogr.am', - 'slideshare.net' - ]) - ); + const minimumAllowlist = self.options.minimumAllowlist || [ + ...self.oembetter.suggestedAllowlist, + 'wufoo.com', + 'infogr.am', + 'slideshare.net' + ]; + + self.oembetter.allowlist(minimumAllowlist.concat(self.options.allowlist || [])); + + const minimumEndpoints = self.options.minimumEndpoints || self.oembetter.suggestedEndpoints; self.oembetter.endpoints( - self.oembetter.suggestedEndpoints.concat(self.options.endpoints || []) + minimumEndpoints.concat(self.options.endpoints || []) ); }, // Enhances oembetter to support services better or to support services - // that have no oembed support by default. Called by `afterConstruct`. - // Extend this method to add additional `oembetter` filters. + // that have no oembed support by default. + // + // Extend or override this method to change or add oembetter filters. enhanceOembetter() { require('./lib/youtube.js')(self, self.oembetter); require('./lib/vimeo.js')(self, self.oembetter); require('./lib/wufoo.js')(self, self.oembetter); - require('./lib/infogram.js')(self, self.oembetter); }, // This method fetches the specified URL, determines its best embedded diff --git a/packages/oembetter/index.js b/packages/oembetter/index.js index 6f33de895f..4c3d168def 100644 --- a/packages/oembetter/index.js +++ b/packages/oembetter/index.js @@ -187,41 +187,15 @@ module.exports = function(options) { self.suggestedAllowlist = [ 'youtube.com', 'youtu.be', - 'blip.tv', 'dailymotion.com', 'flickr.com', - 'hulu.com', - 'nfb.ca', - 'qik.com', - 'revision3.com', - 'scribd.com', - 'viddler.com', 'vimeo.com', - 'youtube.com', - 'dotsub.com', - 'yfrog.com', - 'photobucket.com', 'soundcloud.com', - 'instagram.com', 'twitter.com', - 'facebook.com' + 'x.com' ]; self.suggestedEndpoints = [ - { - domain: 'instagram.com', - endpoint: 'http://api.instagram.com/oembed' - }, - { - domain: 'facebook.com', - path: /\/videos\//, - endpoint: 'https://www.facebook.com/plugins/video/oembed.json/' - }, - { - domain: 'facebook.com', - path: /\/posts\//, - endpoint: 'https://www.facebook.com/plugins/post/oembed.json/' - }, { domain: 'vimeo.com', endpoint: 'https://vimeo.com/api/oembed.json' @@ -233,6 +207,14 @@ module.exports = function(options) { { domain: 'youtu.be', endpoint: 'https://www.youtube.com/oembed' + }, + { + domain: 'twitter.com', + endpoint: 'https://publish.twitter.com/oembed' + }, + { + domain: 'x.com', + endpoint: 'https://publish.twitter.com/oembed' } ]; From bc8f7bed381d4462a082976dd92b8d7f28180a95 Mon Sep 17 00:00:00 2001 From: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Date: Thu, 7 May 2026 18:21:28 +0300 Subject: [PATCH 16/56] Layout editable gap (#5397) --- .changeset/seven-emus-vanish.md | 6 + .../widgets/LayoutWidget.astro | 32 +- .../modules/@apostrophecms/i18n/i18n/en.json | 2 + .../@apostrophecms/layout-widget/index.js | 126 ++++- .../apos/components/AposAreaLayoutEditor.vue | 89 ++- .../ui/apos/components/AposGridLayout.vue | 4 +- .../ui/apos/components/AposGridManager.vue | 4 +- .../layout-widget/ui/apos/layout.css | 8 + .../layout-widget/ui/src/layout.css | 2 +- .../layout-widget/views/widget.html | 6 +- .../modules/@apostrophecms/styles/index.js | 16 + .../@apostrophecms/styles/lib/handlers.js | 6 + .../@apostrophecms/styles/lib/methods.js | 93 +++ .../@apostrophecms/styles/lib/presets.js | 17 + .../styles/ui/apos/universal/render.mjs | 11 +- .../modules/@apostrophecms/util/index.js | 4 + .../@apostrophecms/widget-type/index.js | 6 + .../ui/apos/components/AposWidgetEditor.vue | 32 ++ packages/apostrophe/test/layout-widget-gap.js | 530 ++++++++++++++++++ packages/apostrophe/test/styles.js | 50 ++ 20 files changed, 1022 insertions(+), 22 deletions(-) create mode 100644 .changeset/seven-emus-vanish.md create mode 100644 packages/apostrophe/test/layout-widget-gap.js diff --git a/.changeset/seven-emus-vanish.md b/.changeset/seven-emus-vanish.md new file mode 100644 index 0000000000..564a10e231 --- /dev/null +++ b/.changeset/seven-emus-vanish.md @@ -0,0 +1,6 @@ +--- +"@apostrophecms/apostrophe-astro": minor +"apostrophe": minor +--- + +Editors can now control the layout-widget gap through the styles system, both site-wide via a global `layoutGap` preset and per widget via a `gap` styles field. New Layout widget option `className` allows for additional CSS class names to be added to the widget Grid container. diff --git a/packages/apostrophe-astro/widgets/LayoutWidget.astro b/packages/apostrophe-astro/widgets/LayoutWidget.astro index ec25dfaa5b..596cbecffa 100644 --- a/packages/apostrophe-astro/widgets/LayoutWidget.astro +++ b/packages/apostrophe-astro/widgets/LayoutWidget.astro @@ -2,6 +2,31 @@ const { widget, options } = Astro.props; import AposArea from "../components/AposArea.astro"; import LayoutColumn from "./LayoutColumn.astro"; + +const gapMeta = widget._options || {}; +const resolvedGap = options._gap ?? gapMeta._gap ?? null; +const gapHasGlobal = options._gapHasGlobal ?? gapMeta._gapHasGlobal ?? false; + +// Resolve the inline `--grid-gap` value +const hasWidgetGap = resolvedGap != null; +const omitGridGap = !hasWidgetGap && gapHasGlobal; +const gridGap = hasWidgetGap + ? resolvedGap + : omitGridGap + ? undefined + : options.gap || "0"; + +// Mirror the Nunjucks `parentOptionsForArea` helper so the in-place +// layout editor (AposAreaLayoutEditor) sees the same `gap` signal in +// its `data-parent-options` JSON as it would on the Nunjucks side: +// - widget value present → carries through (string with unit). +// - widget value absent + global enabled → `gap: null` (signal omit). +// - otherwise → no `gap` key (use the static module default / BC). +const parentOptions = { + ...options, + widgetId: widget._id, + ...(hasWidgetGap ? { gap: resolvedGap } : omitGridGap ? { gap: null } : {}), +}; --- diff --git a/packages/apostrophe/modules/@apostrophecms/i18n/i18n/en.json b/packages/apostrophe/modules/@apostrophecms/i18n/i18n/en.json index d2ae0d8af6..e6b5ed913d 100644 --- a/packages/apostrophe/modules/@apostrophecms/i18n/i18n/en.json +++ b/packages/apostrophe/modules/@apostrophecms/i18n/i18n/en.json @@ -643,6 +643,8 @@ "styleGradientAngle": "Angle", "styleGradientEnd": "End Color", "styleGradientStart": "Start Color", + "styleLayoutGap": "Layout Gap", + "styleLayoutGapHelp": "Sets the spacing between columns inside layout sections across the site.", "styleLeft": "Left", "styleMargin": "Margin", "styleOverlayColor": "Overlay Color", diff --git a/packages/apostrophe/modules/@apostrophecms/layout-widget/index.js b/packages/apostrophe/modules/@apostrophecms/layout-widget/index.js index 71377222d0..b6d2060bd7 100644 --- a/packages/apostrophe/modules/@apostrophecms/layout-widget/index.js +++ b/packages/apostrophe/modules/@apostrophecms/layout-widget/index.js @@ -19,6 +19,10 @@ module.exports = { gap: '1.5rem', defaultCellHorizontalAlignment: null, defaultCellVerticalAlignment: null, + // Extra class name(s) to append to the rendered layout-widget area + // wrapper, in addition to the built-in `layout-widget` class. Accepts + // a string of space-separated class names. + className: '', injectStyles: true, minifyStyles: true }, @@ -98,6 +102,25 @@ module.exports = { validateAndIdentifyTypes() { const { column } = self.validateAndIdentifyTypes(); self.columnWidgetName = column; + }, + // Detect the widget-style "gap" field (any styles field whose + // CSS `property` resolves to `gap`). Only the first match is used. + // Also detect whether the @apostrophecms/styles module has a + // site-wide layout gap field configured (via the `layoutGap` + // preset / `layoutGapDefault: true` marker). + detectGapFields() { + const widgetGapFields = self.apos.styles + .fieldsWithProperty(self.schema, 'gap'); + if (widgetGapFields.length > 1) { + self.apos.util.warn( + `[${self.__meta.name}] Multiple style fields produce the ` + + `CSS \`gap\` property (${widgetGapFields.join(', ')}). ` + + 'Only the first one will be honoured as the widget-scope gap.' + ); + } + self.widgetGapFieldName = widgetGapFields[0] || null; + self.globalGapEnabled = !!self.apos.modules['@apostrophecms/styles'] + ?.layoutGapFieldName; } } }; @@ -118,6 +141,17 @@ module.exports = { defaultCellHorizontalAlignment: self.options.defaultCellHorizontalAlignment, defaultCellVerticalAlignment: self.options.defaultCellVerticalAlignment }, + widgetGapFieldName: self.widgetGapFieldName || null, + widgetGapFieldUnit: self.widgetGapFieldName + ? (self.apos.styles + .getFieldByPath(self.schema, self.widgetGapFieldName) + ?.unit || '') + : '', + globalGapEnabled: !!self.globalGapEnabled, + // Opt-in flag read by the generic widget editor: when true, + // it broadcasts `apos-widget-live-preview` events on the apos bus + // during the style-only fast path. + subscribesToLivePreview: !!self.widgetGapFieldName, columnWidgetName: self.columnWidgetName }; }, @@ -132,6 +166,7 @@ module.exports = { defaultCellHorizontalAlignment, defaultCellVerticalAlignment } = self.options; + const widgetGap = self.resolveWidgetGap(widget); return { ..._super(widget, { scene }), columns, @@ -141,13 +176,53 @@ module.exports = { tablet, gap, defaultCellHorizontalAlignment, - defaultCellVerticalAlignment + defaultCellVerticalAlignment, + _gap: widgetGap, + _gapHasGlobal: !!self.globalGapEnabled }; } }; }, methods(self) { return { + // Resolve the widget-scope gap for a widget instance, if this + // module declares a styles field with `property: 'gap'`. Returns + // the resolved value (with unit, when applicable). When the + // widget has no explicit value, falls back to the field's `def`. + // Returns `null` only when no widget gap field is + // configured, no widget value is set, and no `def` is declared + // on the field. + resolveWidgetGap(widget) { + if (!self.widgetGapFieldName || !widget) { + return null; + } + const field = self.apos.styles.getFieldByPath( + self.schema, self.widgetGapFieldName + ); + let value = self.apos.util.get(widget, self.widgetGapFieldName); + if (value === null || value === undefined || value === '') { + if (field?.def === null || field?.def === undefined || field?.def === '') { + return null; + } + value = field.def; + } + if (field?.unit && typeof value !== 'string') { + return `${value}${field.unit}`; + } + return value; + }, + // Determine whether the inline `--grid-gap` CSS variable should + // be omitted on the grid container so the global cascade + // (`var(--apos-layout-gap, …)`) can take effect. The inline var + // is omitted whenever: + // - no widget-scope gap value is set, AND + // - the global layout-gap field is configured. + shouldOmitInlineGap(widget, global) { + if (!self.globalGapEnabled) { + return false; + } + return self.resolveWidgetGap(widget) === null; + }, publicCssNodes(req) { return [ { @@ -277,11 +352,13 @@ module.exports = { const mobileBreakpointPlus = mobileBreakpoint + 1; const tabletBreakpoint = self.options.tablet?.breakpoint || 1024; const tabletBreakpointPlus = tabletBreakpoint + 1; + const gapDefault = self.options.gap || '0'; cssContent = cssContent .replace(/\{\$mobile\}/g, mobileBreakpoint) .replace(/\{\$mobile-plus\}/g, mobileBreakpointPlus) .replace(/\{\$tablet\}/g, tabletBreakpoint) - .replace(/\{\$tablet-plus\}/g, tabletBreakpointPlus); + .replace(/\{\$tablet-plus\}/g, tabletBreakpointPlus) + .replace(/\{\$gap-default\}/g, gapDefault); return self.processCss(cssContent, scene); }, @@ -371,6 +448,51 @@ module.exports = { (a.order ?? 0) - (b.order ?? 0) ); return items[items.length - 1]._id; + }, + // Compute the `--grid-gap: ;` declaration to inline on the + // grid container, or an empty string when the cascade should + // resolve gap via `var(--apos-layout-gap, …)` instead. + // Honours the priority order: + // 1. Widget-style gap (when set on this widget instance). + // 2. Static module option (BC) — when no global gap field is + // configured, or it has no value. + // 3. Otherwise, omit the inline var so the global cascade wins. + // Must be invoked via the widget's own module namespace — + // `apos.modules[data.manager.__meta.name].gapInlineCss(...)` — + // so that `self` resolves to the actual subclass and picks up + // its `widgetGapFieldName` / `globalGapEnabled`. + gapInlineCss(widget, options, global) { + const widgetGap = self.resolveWidgetGap(widget); + if (widgetGap !== null) { + return ` --grid-gap: ${widgetGap};`; + } + if (self.shouldOmitInlineGap(widget, global)) { + return ''; + } + const fallback = (options && options.gap) || self.options.gap || '0'; + return ` --grid-gap: ${fallback};`; + }, + // Build the `aposParentOptions` payload passed by the rendered + // layout-widget area to the in-place editor (AposAreaLayoutEditor). + // Includes the widget's resolved gap (from its `gap` styles field, + // when present) so the live editor's grid container reflects the + // saved per-widget value rather than only the static module + // option. Sets `gap: null` to signal the editor to omit + // `--grid-gap` so the global cascade resolves it through + // `:root { --apos-layout-gap }`. Must be invoked via the widget's + // own module namespace, like `gapInlineCss`. + parentOptionsForArea(widget, options, global) { + const opts = { + ...(options || {}), + widgetId: widget._id + }; + const widgetGap = self.resolveWidgetGap(widget); + if (widgetGap !== null) { + opts.gap = widgetGap; + } else if (self.shouldOmitInlineGap(widget, global)) { + opts.gap = null; + } + return opts; } }; } diff --git a/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposAreaLayoutEditor.vue b/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposAreaLayoutEditor.vue index 9f4f2387d4..9e6d4bd4d8 100644 --- a/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposAreaLayoutEditor.vue +++ b/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposAreaLayoutEditor.vue @@ -122,6 +122,7 @@ '), ''); + assert.equal(sanitizeHtml('<img src=x onerror=alert(1)>'), ''); + assert.equal(sanitizeHtml('<svg><script>alert(1)</script></svg>'), ''); + assert.equal(sanitizeHtml('before<script>alert(1)</script>after'), 'beforeafter'); + }); it('should retain the content of fibble elements by default', function() { assert.equal(sanitizeHtml('Nifty

Paragraph

'), 'Nifty

Paragraph

'); }); From 5d1a02876b9305afd585e7020f8aa4eba1c6ddd0 Mon Sep 17 00:00:00 2001 From: Tom Boutell Date: Wed, 13 May 2026 08:56:52 -0400 Subject: [PATCH 21/56] Merge commit from fork * secure the link URL field of image widgets * credit --- .changeset/pink-buckets-write.md | 7 ++ .../@apostrophecms/image-widget/index.js | 30 ++++- packages/apostrophe/package.json | 2 +- packages/launder/index.js | 113 ++++++++++++------ packages/launder/test/test.js | 70 +++++++++++ packages/sanitize-html/index.js | 49 ++------ packages/sanitize-html/package.json | 1 + 7 files changed, 194 insertions(+), 78 deletions(-) create mode 100644 .changeset/pink-buckets-write.md diff --git a/.changeset/pink-buckets-write.md b/.changeset/pink-buckets-write.md new file mode 100644 index 0000000000..9333df426d --- /dev/null +++ b/.changeset/pink-buckets-write.md @@ -0,0 +1,7 @@ +--- +"sanitize-html": patch +"apostrophe": patch +"launder": patch +--- + +Security: launder now uses and exports the best available naughtyHref function for detecting malicious URLs. sanitize-html now depends on it, and apostrophe now uses type: 'url' for the link URL field of image widgets, which leverages it. Prior to this fix, it was possible for any user with editing privileges, including a contributor, to trigger arbitrary JavaScript via a javascript: URL in the link URL field of an image widget. A migration has been included to strip any such malicious URLs already present in the database. All users of apostrophe are encouraged to upgrade to get this security fix. Thanks to [Muhammad Uwais](https://github.com/MuhammadUwais) for reporting the issue. diff --git a/packages/apostrophe/modules/@apostrophecms/image-widget/index.js b/packages/apostrophe/modules/@apostrophecms/image-widget/index.js index 43fe175055..ebe2175625 100644 --- a/packages/apostrophe/modules/@apostrophecms/image-widget/index.js +++ b/packages/apostrophe/modules/@apostrophecms/image-widget/index.js @@ -102,7 +102,7 @@ module.exports = { linkHref: { label: 'apostrophe:url', help: 'apostrophe:linkHrefHelp', - type: 'string', + type: 'url', required: true, if: { linkTo: '_url' @@ -147,6 +147,7 @@ module.exports = { self.showPlaceholder = self.options.placeholder !== false; self.options.placeholder = true; self.determineBestAssetUrl('placeholder'); + self.addCleanLinkHrefMigration(); }, handlers(self) { return { @@ -159,6 +160,33 @@ module.exports = { }, methods(self) { return { + // Clear link hrefs that use dangerous URL schemes (e.g. + // `javascript:`) and may have been stored before the linkHref + // schema field was changed to `type: 'url'`. Registered per + // module instance so subclasses of this widget are migrated + // under their own widget type and migration name. + addCleanLinkHrefMigration() { + self.apos.migration.add( + `${self.__meta.name}:clean-naughty-link-href`, + async () => { + await self.apos.migration.eachWidget({}, async (doc, widget, dotPath) => { + if (widget.type !== self.name) { + return; + } + if (!widget.linkHref) { + return; + } + if (!self.apos.launder.naughtyHref(widget.linkHref)) { + return; + } + await self.apos.doc.db.updateOne( + { _id: doc._id }, + { $set: { [`${dotPath}.linkHref`]: '' } } + ); + }); + } + ); + }, validateAndAddSchemaLabels() { const linkWithType = self.options.linkWithType; diff --git a/packages/apostrophe/package.json b/packages/apostrophe/package.json index a01079b6d2..f33c12137c 100644 --- a/packages/apostrophe/package.json +++ b/packages/apostrophe/package.json @@ -93,7 +93,7 @@ "is-wsl": "^2.2.0", "jsdom": "^24.1.0", "klona": "^2.0.4", - "launder": "^1.4.0", + "launder": "workspace:^", "lodash": "^4.18.1", "mini-css-extract-plugin": "^1.6.0", "minimatch": "^3.1.4", diff --git a/packages/launder/index.js b/packages/launder/index.js index 3393ce17a1..0cb59390c5 100644 --- a/packages/launder/index.js +++ b/packages/launder/index.js @@ -1,5 +1,64 @@ const dayjs = require('dayjs'); +// Strip characters browsers ignore inside URLs (control chars and +// embedded HTML comments) that are commonly used to sneak XSS +// payloads past simple scheme checks. +function cleanHref(href) { + // Browsers ignore character codes of 32 (space) and below in a surprising + // number of situations. Start reading here: + // https://www.owasp.org/index.php/XSS_Filter_Evasion_Cheat_Sheet#Embedded_tab + // eslint-disable-next-line no-control-regex + href = href.replace(/[\x00-\x20]+/g, ''); + // Clobber any comments in URLs, which the browser might + // interpret inside an XML data island, allowing + // a javascript: URL to be snuck through + while (true) { + const firstIndex = href.indexOf('', firstIndex + 4); + if (lastIndex === -1) { + break; + } + href = href.substring(0, firstIndex) + href.substring(lastIndex + 3); + } + return href; +} + +// Returns true if `href` should be rejected as unsafe. Used to block +// `javascript:`, `data:` and other dangerous schemes both when +// validating URL fields and when sanitizing HTML attribute values. +// +// `options.allowedSchemes` is an array of permitted URL schemes +// (defaults to the schemes long permitted by `launder().url`). +// `options.allowProtocolRelative` controls whether protocol-relative +// URLs (e.g. `//example.com`) are permitted. Defaults to true. +function naughtyHref(href, options) { + options = options || {}; + const allowedSchemes = options.allowedSchemes || + [ 'http', 'https', 'ftp', 'mailto', 'tel', 'sms' ]; + const allowProtocolRelative = (options.allowProtocolRelative !== false); + if (typeof href !== 'string') { + return false; + } + href = cleanHref(href); + // Case insensitive so we don't get faked out by JAVASCRIPT #1 + // Allow more characters after the first so we don't get faked + // out by certain schemes browsers accept + const matches = href.match(/^([a-zA-Z][a-zA-Z0-9.\-+]*):/); + if (!matches) { + // Protocol-relative URL starting with any combination of '/' and '\' + if (href.match(/^[/\\]{2}/)) { + return !allowProtocolRelative; + } + // No scheme + return false; + } + const scheme = matches[1].toLowerCase(); + return allowedSchemes.indexOf(scheme) === -1; +} + module.exports = function(options) { const self = {}; self.options = options || {}; @@ -91,18 +150,27 @@ module.exports = function(options) { return i; }; + self.naughtyHref = naughtyHref; + self.url = function(s, def, httpsFix) { s = self.string(s, def); // Allow the default to be undefined, null, false, etc. if (s === def) { return s; } - s = fixUrl(s); - if (s === null) { + // Strip characters browsers ignore that are commonly used to + // smuggle dangerous schemes past validation. + s = cleanHref(s); + // Reject naughty URLs (e.g. `javascript:`, `data:`) before fixUrl + // runs. Otherwise fixUrl could mask such schemes by prepending + // `https://` to anything containing a `.`, turning + // `javascript:alert(document.domain)` into a syntactically odd + // but accepted `https://...` URL. + if (naughtyHref(s)) { return def; } - s = naughtyHref(s); - if (s === true) { + s = fixUrl(s); + if (s === null) { return def; } return s; @@ -119,39 +187,6 @@ module.exports = function(options) { return null; } }; - - function naughtyHref(href) { - // Browsers ignore character codes of 32 (space) and below in a surprising - // number of situations. Start reading here: - // https://www.owasp.org/index.php/XSS_Filter_Evasion_Cheat_Sheet#Embedded_tab - // eslint-disable-next-line no-control-regex - href = href.replace(/[\x00-\x20]+/g, ''); - // Clobber any comments in URLs, which the browser might - // interpret inside an XML data island, allowing - // a javascript: URL to be snuck through - while (true) { - const firstIndex = href.indexOf('', firstIndex + 4); - if (lastIndex === -1) { - break; - } - href = href.substring(0, firstIndex) + href.substring(lastIndex + 3); - } - // Case insensitive so we don't get faked out by JAVASCRIPT #1 - // Allow more characters after the first so we don't get faked - // out by certain schemes browsers accept - const matches = href.match(/^([a-zA-Z]+):/); - if (!matches) { - // No scheme = no way to inject js (right?) - return href; - } - const scheme = matches[1].toLowerCase(); - - return (![ 'http', 'https', 'ftp', 'mailto', 'tel', 'sms' ].includes(scheme)) ? true : href; - } }; self.select = function(s, choices, def) { @@ -497,3 +532,7 @@ module.exports = function(options) { }; return self; }; + +// Expose `naughtyHref` statically so callers (notably sanitize-html) +// can use it without instantiating a launder. +module.exports.naughtyHref = naughtyHref; diff --git a/packages/launder/test/test.js b/packages/launder/test/test.js index 75c7542e23..e491b6c9c2 100644 --- a/packages/launder/test/test.js +++ b/packages/launder/test/test.js @@ -250,6 +250,76 @@ describe('launder', function() { it('should add https:// when missing if httpsFix is true', function() { assert(launder.url('www.apostrophenow.org', null, true) === 'https://www.apostrophenow.org'); }); + it('should reject javascript: even when the URL contains a dot', function() { + // Without rejecting before fixUrl, the dot in `document.domain` + // causes fixUrl to prepend `https://`, masking the dangerous + // scheme rather than discarding it. + assert(launder.url('javascript:alert(document.domain)', 'http://www.apostrophenow.org') === 'http://www.apostrophenow.org'); + }); + it('should reject javascript: with mixed case', function() { + assert(launder.url('JaVaScRiPt:alert(1)', 'http://www.apostrophenow.org') === 'http://www.apostrophenow.org'); + }); + it('should reject javascript: smuggled past simple checks with control characters', function() { + assert(launder.url('java\tscript:alert(1)', 'http://www.apostrophenow.org') === 'http://www.apostrophenow.org'); + }); + it('should reject javascript: smuggled past simple checks with HTML comments', function() { + assert(launder.url('javascript:alert(1)', 'http://www.apostrophenow.org') === 'http://www.apostrophenow.org'); + }); + it('should reject data: urls', function() { + assert(launder.url('data:text/html,', 'http://www.apostrophenow.org') === 'http://www.apostrophenow.org'); + }); + it('should preserve relative urls beginning with /', function() { + assert(launder.url('/about') === '/about'); + }); + it('should preserve protocol-relative urls', function() { + assert(launder.url('//www.apostrophenow.org') === '//www.apostrophenow.org'); + }); + }); + + describe('naughtyHref', function() { + it('should report javascript: as naughty', function() { + assert(launder.naughtyHref('javascript:alert(1)') === true); + }); + it('should report data: as naughty', function() { + assert(launder.naughtyHref('data:text/html,') === true); + }); + it('should report mixed-case javascript: as naughty', function() { + assert(launder.naughtyHref('JaVaScRiPt:alert(1)') === true); + }); + it('should report control-character-smuggled javascript: as naughty', function() { + assert(launder.naughtyHref('java\tscript:alert(1)') === true); + }); + it('should report HTML-comment-smuggled javascript: as naughty', function() { + assert(launder.naughtyHref('javascript:alert(1)') === true); + }); + it('should accept allowed schemes', function() { + assert(launder.naughtyHref('https://example.com') === false); + assert(launder.naughtyHref('http://example.com') === false); + assert(launder.naughtyHref('ftp://example.com') === false); + assert(launder.naughtyHref('mailto:foo@example.com') === false); + assert(launder.naughtyHref('tel:1234') === false); + assert(launder.naughtyHref('sms:1234') === false); + }); + it('should accept relative URLs', function() { + assert(launder.naughtyHref('/about') === false); + assert(launder.naughtyHref('about') === false); + assert(launder.naughtyHref('#fragment') === false); + }); + it('should accept protocol-relative URLs by default', function() { + assert(launder.naughtyHref('//example.com') === false); + }); + it('should reject protocol-relative URLs when allowProtocolRelative is false', function() { + assert(launder.naughtyHref('//example.com', { allowProtocolRelative: false }) === true); + }); + it('should respect a custom allowedSchemes list', function() { + assert(launder.naughtyHref('https://example.com', { allowedSchemes: [ 'http' ] }) === true); + assert(launder.naughtyHref('http://example.com', { allowedSchemes: [ 'http' ] }) === false); + }); + it('should return false for non-string input', function() { + assert(launder.naughtyHref(undefined) === false); + assert(launder.naughtyHref(null) === false); + assert(launder.naughtyHref(42) === false); + }); }); describe('select', function() { diff --git a/packages/sanitize-html/index.js b/packages/sanitize-html/index.js index eec6ca628c..6a67683868 100644 --- a/packages/sanitize-html/index.js +++ b/packages/sanitize-html/index.js @@ -4,6 +4,7 @@ const { isPlainObject } = require('is-plain-object'); const deepmerge = require('deepmerge'); const parseSrcset = require('parse-srcset'); const { parse: postcssParse } = require('postcss'); +const { naughtyHref: launderNaughtyHref } = require('launder'); // Tags that can conceivably represent stand-alone media. const mediaTags = [ 'img', 'audio', 'video', 'picture', 'svg', @@ -731,45 +732,15 @@ function sanitizeHtml(html, options, _recursing) { } function naughtyHref(name, href) { - // Browsers ignore character codes of 32 (space) and below in a surprising - // number of situations. Start reading here: - // https://www.owasp.org/index.php/XSS_Filter_Evasion_Cheat_Sheet#Embedded_tab - // eslint-disable-next-line no-control-regex - href = href.replace(/[\x00-\x20]+/g, ''); - // Clobber any comments in URLs, which the browser might - // interpret inside an XML data island, allowing - // a javascript: URL to be snuck through - while (true) { - const firstIndex = href.indexOf('', firstIndex + 4); - if (lastIndex === -1) { - break; - } - href = href.substring(0, firstIndex) + href.substring(lastIndex + 3); - } - // Case insensitive so we don't get faked out by JAVASCRIPT #1 - // Allow more characters after the first so we don't get faked - // out by certain schemes browsers accept - const matches = href.match(/^([a-zA-Z][a-zA-Z0-9.\-+]*):/); - if (!matches) { - // Protocol-relative URL starting with any combination of '/' and '\' - if (href.match(/^[/\\]{2}/)) { - return !options.allowProtocolRelative; - } - - // No scheme - return false; - } - const scheme = matches[1].toLowerCase(); - - if (has(options.allowedSchemesByTag, name)) { - return options.allowedSchemesByTag[name].indexOf(scheme) === -1; - } - - return !options.allowedSchemes || options.allowedSchemes.indexOf(scheme) === -1; + // Resolve the per-tag scheme allowlist if one is configured for + // this tag, otherwise fall back to the global allowedSchemes. + const allowedSchemes = has(options.allowedSchemesByTag, name) + ? options.allowedSchemesByTag[name] + : (options.allowedSchemes || []); + return launderNaughtyHref(href, { + allowedSchemes, + allowProtocolRelative: options.allowProtocolRelative + }); } function parseUrl(value) { diff --git a/packages/sanitize-html/package.json b/packages/sanitize-html/package.json index d6ffe4fdc1..fe401c0764 100644 --- a/packages/sanitize-html/package.json +++ b/packages/sanitize-html/package.json @@ -30,6 +30,7 @@ "escape-string-regexp": "^4.0.0", "htmlparser2": "^10.1.0", "is-plain-object": "^5.0.0", + "launder": "workspace:^", "parse-srcset": "^1.0.2", "postcss": "^8.3.11" }, From 41670a39e681449758ca82a071e9bb84c265048a Mon Sep 17 00:00:00 2001 From: Tom Boutell Date: Wed, 13 May 2026 08:57:42 -0400 Subject: [PATCH 22/56] Merge commit from fork --- .changeset/busy-spies-enter.md | 5 + .../modules/@apostrophecms/login/index.js | 28 ++-- packages/apostrophe/test/login.js | 123 +++++++++++++++++- 3 files changed, 140 insertions(+), 16 deletions(-) create mode 100644 .changeset/busy-spies-enter.md diff --git a/.changeset/busy-spies-enter.md b/.changeset/busy-spies-enter.md new file mode 100644 index 0000000000..cc922b7b1c --- /dev/null +++ b/.changeset/busy-spies-enter.md @@ -0,0 +1,5 @@ +--- +"apostrophe": patch +--- + +Security fix: the password reset request feature will refuse to operate unless the baseUrl option or the APOS_BASE_URL environment variable has been set (note that this is automatic in multisite projects). This fix is necessary to prevent a vulnerability that can be used to convince ApostropheCMS to send emails containing links to other sites. It is only a vulnerability if you have enabled the passwordReset: true option for the login module. Thanks to [SPIDY](https://github.com/Mujahidkhan525) for reporting the issue. diff --git a/packages/apostrophe/modules/@apostrophecms/login/index.js b/packages/apostrophe/modules/@apostrophecms/login/index.js index 224e22d0fb..bdcf934859 100644 --- a/packages/apostrophe/modules/@apostrophecms/login/index.js +++ b/packages/apostrophe/modules/@apostrophecms/login/index.js @@ -295,7 +295,17 @@ module.exports = { async resetRequest(req) { const MIN_RESPONSE_TIME = 2000; const startTime = Date.now(); - const site = (req.headers.host || '').replace(/:\d+$/, ''); + // Refuse to construct reset URLs from the request's Host header, + // which is attacker-controlled. Operators must configure the + // baseUrl option (or the APOS_BASE_URL environment variable) so + // the link in the reset email points to the real site rather than + // wherever the attacker aimed their crafted request. + if (!self.apos.baseUrl) { + throw self.apos.error( + 'invalid', + 'The baseUrl option (or APOS_BASE_URL environment variable) must be configured to enable password reset' + ); + } const email = self.apos.launder.string(req.body.email); if (!email.length) { throw self.apos.error('invalid', req.t('apostrophe:loginResetEmailRequired')); @@ -319,22 +329,10 @@ module.exports = { user.passwordReset = reset; user.passwordResetAt = new Date(); await self.apos.user.update(req, user, { permissions: false }); - let port = (req.headers.host || '').split(':')[1]; - if (!port || [ '80', '443' ].includes(port)) { - port = ''; - } else { - port = `:${port}`; - } - const parsed = new URL( - req.absoluteUrl, - self.apos.baseUrl - ? undefined - : `${req.protocol}://${req.hostname}${port}` - ); - parsed.pathname = self.login(); - parsed.search = '?'; + const parsed = new URL(self.login(), self.apos.baseUrl); parsed.searchParams.append('reset', reset); parsed.searchParams.append('email', user.email); + const site = parsed.hostname; try { await self.email(req, 'passwordResetEmail', { user, diff --git a/packages/apostrophe/test/login.js b/packages/apostrophe/test/login.js index 82ec345ec6..877c2207dc 100644 --- a/packages/apostrophe/test/login.js +++ b/packages/apostrophe/test/login.js @@ -11,6 +11,7 @@ describe('Login', function () { before(async function () { apos = await t.create({ root: module, + baseUrl: 'http://localhost:3000', modules: { '@apostrophecms/express': { options: { @@ -602,8 +603,14 @@ describe('Login', function () { assert.equal(opts.to, user.email); assert(opts.subject); - // Safe the token for the reset tests + // The reset URL must be derived from apos.baseUrl, not from the + // request's Host header, to prevent host header injection attacks + // that could leak the reset token to an attacker-controlled domain. const url = new URL(data.url); + assert.equal(`${url.protocol}//${url.host}`, apos.baseUrl); + assert.equal(data.site, new URL(apos.baseUrl).hostname); + + // Safe the token for the reset tests resetToken = url.searchParams.get('reset'); } @@ -798,6 +805,56 @@ describe('Login', function () { assert(page.match(/logged in/)); }); + it('should ignore a spoofed Host header on POST /login/reset-request', async function () { + let args; + const orig = apos.login.email; + apos.login.email = (req, ...a) => { + args = a; + }; + + let user = apos.user.newInstance(); + user.title = 'spoofUser'; + user.email = 'spoofUser@example.com'; + user.username = 'spoofUser'; + user.password = 'secret'; + user.role = 'guest'; + user = await apos.user.insert(apos.task.getReq(), user); + + const jar = apos.http.jar(); + await apos.http.get( + '/', + { + jar + } + ); + + await apos.http.post( + '/api/v1/@apostrophecms/login/reset-request', + { + body: { + email: 'spoofUser', + session: true + }, + headers: { + Host: 'evil.attacker.example' + }, + jar + } + ); + + assert(Array.isArray(args)); + const [ , data ] = args; + const url = new URL(data.url); + // The reset URL must reflect the configured apos.baseUrl, not the + // attacker-supplied Host header. + assert.equal(`${url.protocol}//${url.host}`, apos.baseUrl); + assert.notEqual(url.hostname, 'evil.attacker.example'); + assert.equal(data.site, new URL(apos.baseUrl).hostname); + assert.notEqual(data.site, 'evil.attacker.example'); + + apos.login.email = orig; + }); + it('should find user by reset data', async function () { let user = apos.user.newInstance(); user.title = 'getResetUser'; @@ -1053,6 +1110,70 @@ describe('Login', function () { assert.deepEqual(actual, expected); }); }); + + describe('passwordReset without apos.baseUrl', function () { + let apos3; + + this.timeout(20000); + + before(async function () { + apos3 = await t.create({ + root: module, + modules: { + '@apostrophecms/login': { + options: { + passwordReset: true, + environmentLabel: 'test' + } + } + } + }); + }); + + after(function () { + return t.destroy(apos3); + }); + + it('should reject reset-request when apos.baseUrl is not configured', async function () { + assert(!apos3.baseUrl); + + const user = apos3.user.newInstance(); + user.title = 'noBaseUrlUser'; + user.username = 'noBaseUrlUser'; + user.email = 'noBaseUrlUser@example.com'; + user.password = 'secret'; + user.role = 'guest'; + await apos3.user.insert(apos3.task.getReq(), user); + + const jar = apos3.http.jar(); + await apos3.http.get('/', { jar }); + + let emailed = false; + const orig = apos3.login.email; + apos3.login.email = () => { + emailed = true; + }; + + await assert.rejects(() => apos3.http.post( + '/api/v1/@apostrophecms/login/reset-request', + { + body: { + email: 'noBaseUrlUser', + session: true + }, + jar + } + ), { + status: 400 + }); + + // Confirm no reset email was sent — the request must be refused + // before any token is generated or message dispatched. + assert.equal(emailed, false); + + apos3.login.email = orig; + }); + }); }); describe('Case Sensitivity', function() { From 01bacd24ad40d973cad7962e50f1d32ba05130f8 Mon Sep 17 00:00:00 2001 From: Tom Boutell Date: Wed, 13 May 2026 08:57:52 -0400 Subject: [PATCH 23/56] Merge commit from fork --- .changeset/fiery-lights-fetch.md | 5 +++++ packages/cli/lib/commands/create.js | 23 ++++++++++++++--------- packages/cli/lib/util.js | 24 ++++++++++++++++++++++++ packages/cli/package.json | 1 - 4 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 .changeset/fiery-lights-fetch.md diff --git a/.changeset/fiery-lights-fetch.md b/.changeset/fiery-lights-fetch.md new file mode 100644 index 0000000000..3bcf8c1da7 --- /dev/null +++ b/.changeset/fiery-lights-fetch.md @@ -0,0 +1,5 @@ +--- +"@apostrophecms/cli": patch +--- + +Security: passwords and starter kit URLs containing intentionally malicious punctuation cannot be used to run arbitrary shell commands. Because the CLI is only used by developers, this would always have been an "own goal" situation, however this does make the CLI more robust for scripted use. Thanks to [Nitro13urn](https://github.com/VadlaReddySai) for reporting the issue. diff --git a/packages/cli/lib/commands/create.js b/packages/cli/lib/commands/create.js index 8596c3c8bb..c7120ab7c0 100644 --- a/packages/cli/lib/commands/create.js +++ b/packages/cli/lib/commands/create.js @@ -1,13 +1,12 @@ require('shelljs/global'); // Utilities from shelljs -/* globals exec cd rm */ +/* globals cd rm */ const prompts = require('prompts'); const path = require('path'); const util = require('../util'); const config = require('../../config'); const fs = require('fs'); const { stripIndent } = require('common-tags'); -const quote = require('shell-quote').quote; module.exports = function (program) { program @@ -52,7 +51,7 @@ module.exports = function (program) { ); // Clone the boilerplate project - if (exec(`git clone ${boilerplateUrl} ${shortName}`).code !== 0) { + if ((await util.spawnSafe('git', [ 'clone', boilerplateUrl, shortName ])).code !== 0) { await util.error('create', 'Error cloning starter code.'); return false; } @@ -175,15 +174,21 @@ module.exports = function (program) { const userTask = '@apostrophecms/user:add'; const appJsPath = resolvePath('app.js'); - let createUserCommand = `node ${appJsPath} ${userTask} admin admin`; + const createUserArgs = [ appJsPath, userTask, 'admin', 'admin' ]; - // Prepend MongoDB URI if provided + const createUserEnv = { ...process.env }; if (options.mongodbUri) { - // Ensure the URI is properly quoted to handle special characters - createUserCommand = `APOS_MONGODB_URI=${quote([ options.mongodbUri ])} ` + createUserCommand; + createUserEnv.APOS_MONGODB_URI = options.mongodbUri; + } + util.log('create', `Creating admin user: node ${createUserArgs.join(' ')}`); + const createUserResult = await util.spawnSafe('node', createUserArgs, { + env: createUserEnv, + input: `${response.pw ?? ''}\n` + }); + if (createUserResult.code !== 0) { + await util.error('create', `Admin user creation exited with code ${createUserResult.code}.`); + return false; } - util.log('create', `Creating admin user with command: ${createUserCommand}`); - exec(`echo "${response.pw}" | ${createUserCommand}`); util.log('create', 'All done! 🎉 Login as "admin" at the /login URL.'); await util.success('create'); diff --git a/packages/cli/lib/util.js b/packages/cli/lib/util.js index 566b3e75fc..a6bdf9f415 100644 --- a/packages/cli/lib/util.js +++ b/packages/cli/lib/util.js @@ -167,6 +167,30 @@ util.secret = function() { return string; }; +// Spawn a child process without invoking a shell, so caller-supplied URLs, +// passwords, etc. cannot be parsed as shell metacharacters. Returns a promise +// resolving to `{ code }`, mirroring the `.code` property on shelljs's `exec` +// result so call sites can use the same shape. Pass `input` to write a string +// to the child's stdin (e.g. an interactive password prompt response); when +// omitted, stdin is inherited from the parent. +util.spawnSafe = function (command, args, { + env, input, cwd +} = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + env: env ?? process.env, + cwd, + stdio: [ input != null ? 'pipe' : 'inherit', 'inherit', 'inherit' ] + }); + child.on('error', reject); + child.on('exit', (code) => resolve({ code })); + if (input != null) { + child.stdin.on('error', reject); + child.stdin.end(input); + } + }); +}; + util.spawnWithSpinner = async function (command, options = { spinnerMessage: 'Processing...', codeMessage: 'Spawn process error code:' diff --git a/packages/cli/package.json b/packages/cli/package.json index 20ff3c1395..c4f9221650 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -26,7 +26,6 @@ "pkginfo": "^0.4.1", "prompts": "^2.3.2", "semver": "^7.3.2", - "shell-quote": "^1.8.1", "shelljs": "^0.10.0", "sync-request": "^6.0.0", "uuid": "^11.1.0" From 8098017bb3f0ed55e8f231011c2924084eb9045d Mon Sep 17 00:00:00 2001 From: Tom Boutell Date: Wed, 13 May 2026 08:58:02 -0400 Subject: [PATCH 24/56] Merge commit from fork * Fixed arbitrary image URL import vulnerability in rich text impport API * additional reporter --- .changeset/solid-boats-send.md | 10 + .../@apostrophecms/rich-text-widget/index.js | 36 ++++ packages/apostrophe/test/rich-text-widget.js | 200 ++++++++++++++++++ 3 files changed, 246 insertions(+) create mode 100644 .changeset/solid-boats-send.md diff --git a/.changeset/solid-boats-send.md b/.changeset/solid-boats-send.md new file mode 100644 index 0000000000..d0f848873f --- /dev/null +++ b/.changeset/solid-boats-send.md @@ -0,0 +1,10 @@ +--- +"apostrophe": patch +--- + +Security: the HTML import feature of the rich text widget no longer permits images to be fetched +from arbitrary hosts. This could be used to probe internal networks, and to exfiltrate images from +internal hosts if their URLs were known. Instead, the `imageImportAllowedHostnames` option of the +`@apostrophecms/rich-text-widget` must be configured to opt into that feature. + +Thanks to [Yiğit Şengezer](https://github.com/yigitsengezer) and [Sainithin0309](https://github.com/Sainithin0309) for reporting this issue. diff --git a/packages/apostrophe/modules/@apostrophecms/rich-text-widget/index.js b/packages/apostrophe/modules/@apostrophecms/rich-text-widget/index.js index f394d06479..98ca65ef30 100644 --- a/packages/apostrophe/modules/@apostrophecms/rich-text-widget/index.js +++ b/packages/apostrophe/modules/@apostrophecms/rich-text-widget/index.js @@ -128,6 +128,10 @@ module.exports = { defaultData: { content: '' }, className: false, linkWithType: [ '@apostrophecms/any-page-type' ], + // Hostnames from which `` tags in `import.html` may be fetched. + // The list is empty by default, which disables image fetching during + // rich text HTML import. Add hostnames here to opt in. + imageImportAllowedHostnames: [], tableOptions: { resizable: true, handleWidth: 10, @@ -472,6 +476,31 @@ module.exports = { return widget.content; }, + // Return the configured allowlist of hostnames from which images may + // be fetched during a rich text widget HTML import, normalized to + // lowercase. Returns an empty array if the option is unset, in which + // case any image fetch attempted during import is rejected. + getImageImportAllowedHostnames() { + const list = self.options.imageImportAllowedHostnames; + if (!Array.isArray(list)) { + return []; + } + return list + .filter(entry => typeof entry === 'string' && entry.length > 0) + .map(entry => entry.toLowerCase()); + }, + + // Determine whether the given URL's hostname is permitted by the + // import allowlist. The protocol is also restricted to http/https + // to prevent fetches via file:, data:, or other schemes. + isImageImportHostnameAllowed(url, allowedHostnames) { + if (!url || (url.protocol !== 'http:' && url.protocol !== 'https:')) { + return false; + } + const hostname = (url.hostname || '').toLowerCase(); + return allowedHostnames.includes(hostname); + }, + // Handle relationships to permalinks and inline images async load(req, widgets) { try { @@ -1057,6 +1086,7 @@ module.exports = { if (input.import.baseUrl && ((typeof input.import.html) !== 'string')) { throw self.apos.error('invalid', 'If present, import.baseUrl must be a string'); } + const allowedHostnames = self.getImageImportAllowedHostnames(); const $ = cheerio.load(input.import.html); const $images = $('img'); // Build an array of cheerio objects because @@ -1071,6 +1101,12 @@ module.exports = { const src = $image.attr('src'); const alt = $image.attr('alt') && self.apos.util.escapeHtml($image.attr('alt')); const url = new URL(src, input.import.baseUrl || self.apos.baseUrl); + if (!self.isImageImportHostnameAllowed(url, allowedHostnames)) { + throw self.apos.error( + 'forbidden', + `Refusing to import image from disallowed hostname "${url.hostname}". Add it to the \`imageImportAllowedHostnames\` option of the \`@apostrophecms/rich-text-widget\` module to allow it.` + ); + } const res = await fetch(url); if (res.status >= 400) { self.apos.util.warn(`Error ${res.status} while importing ${src}, ignoring image`); diff --git a/packages/apostrophe/test/rich-text-widget.js b/packages/apostrophe/test/rich-text-widget.js index 05581e76ac..5a82caa7c2 100644 --- a/packages/apostrophe/test/rich-text-widget.js +++ b/packages/apostrophe/test/rich-text-widget.js @@ -293,4 +293,204 @@ describe('Rich Text Widget', function () { assert(text2.includes('src="/uploads/attachments/attachment-1-attachment-1.max.jpg" alt="Updated Test Image 1"')); assert(text2.includes('src="/uploads/attachments/attachment-2-attachment-2.max.jpg" alt="Updated Test Image 2"')); }); + + describe('image import allowlist (SSRF mitigation)', function () { + let originalConsoleError; + + before(function () { + // The sanitize method intentionally console.errors thrown errors to + // surface stack traces during development; silence that noise during + // these negative tests. + // eslint-disable-next-line no-console + originalConsoleError = console.error; + // eslint-disable-next-line no-console + console.error = () => {}; + }); + + after(function () { + // eslint-disable-next-line no-console + console.error = originalConsoleError; + }); + + it('allows rich text HTML import without `` tags even with no allowlist configured', async function () { + apos = await t.create({ + root: module, + modules: {} + }); + + const manager = apos.modules['@apostrophecms/rich-text-widget']; + const req = apos.task.getReq(); + + const output = await manager.sanitize(req, { + type: '@apostrophecms/rich-text', + content: '

seed

', + import: { + html: '

Hello world

' + } + }, {}); + + assert.match(output.content, /Hello/); + }); + + it('rejects an image fetch for a hostname not in the allowlist and never calls fetch', async function () { + apos = await t.create({ + root: module, + modules: { + '@apostrophecms/rich-text-widget': { + options: { + imageImportAllowedHostnames: [ 'images.example.com' ] + } + } + } + }); + + const manager = apos.modules['@apostrophecms/rich-text-widget']; + const req = apos.task.getReq(); + + const originalFetch = global.fetch; + let fetchCalled = false; + global.fetch = async () => { + fetchCalled = true; + throw new Error('fetch should not be called'); + }; + + try { + await assert.rejects( + () => manager.sanitize(req, { + type: '@apostrophecms/rich-text', + content: '

seed

', + import: { + html: '', + baseUrl: 'http://127.0.0.1:7777' + } + }, {}), + (err) => { + assert.equal(err.name, 'forbidden'); + assert.match(err.message, /127\.0\.0\.1/); + assert.match(err.message, /imageImportAllowedHostnames/); + return true; + } + ); + } finally { + global.fetch = originalFetch; + } + + assert.equal(fetchCalled, false); + }); + + it('rejects rich text HTML import for non-http(s) protocols even if hostname matches', async function () { + apos = await t.create({ + root: module, + modules: { + '@apostrophecms/rich-text-widget': { + options: { + imageImportAllowedHostnames: [ 'localhost' ] + } + } + } + }); + + const manager = apos.modules['@apostrophecms/rich-text-widget']; + const req = apos.task.getReq(); + + await assert.rejects( + () => manager.sanitize(req, { + type: '@apostrophecms/rich-text', + content: '

seed

', + import: { + html: '', + baseUrl: 'file://localhost' + } + }, {}), + (err) => { + assert.equal(err.name, 'forbidden'); + return true; + } + ); + }); + + it('proceeds to fetch when import URL hostname is on the allowlist', async function () { + apos = await t.create({ + root: module, + modules: { + '@apostrophecms/rich-text-widget': { + options: { + imageImportAllowedHostnames: [ 'images.example.com' ] + } + } + } + }); + + const manager = apos.modules['@apostrophecms/rich-text-widget']; + const req = apos.task.getReq(); + + const originalFetch = global.fetch; + const fetchedUrls = []; + // Throw a unique marker error from inside fetch. If the allowlist + // passes, the marker error propagates; if the allowlist rejects, we + // would see a `forbidden` error instead. + const marker = new Error('FETCH_REACHED'); + marker.name = 'FetchReached'; + global.fetch = async (url) => { + fetchedUrls.push(url.toString()); + throw marker; + }; + + try { + await assert.rejects( + () => manager.sanitize(req, { + type: '@apostrophecms/rich-text', + content: '

seed

', + import: { + html: '', + baseUrl: 'https://images.example.com' + } + }, {}), + (err) => err.name === 'FetchReached' + ); + } finally { + global.fetch = originalFetch; + } + + assert.deepEqual(fetchedUrls, [ 'https://images.example.com/foo.png' ]); + }); + + it('helper methods reflect option configuration', async function () { + apos = await t.create({ + root: module, + modules: { + '@apostrophecms/rich-text-widget': { + options: { + imageImportAllowedHostnames: [ 'Images.Example.com', '', null, 'cdn.example.com' ] + } + } + } + }); + + const manager = apos.modules['@apostrophecms/rich-text-widget']; + + assert.deepEqual( + manager.getImageImportAllowedHostnames(), + [ 'images.example.com', 'cdn.example.com' ] + ); + + const allowed = manager.getImageImportAllowedHostnames(); + assert.equal( + manager.isImageImportHostnameAllowed(new URL('https://images.example.com/foo.png'), allowed), + true + ); + assert.equal( + manager.isImageImportHostnameAllowed(new URL('https://IMAGES.example.com/foo.png'), allowed), + true + ); + assert.equal( + manager.isImageImportHostnameAllowed(new URL('https://attacker.example.com/foo.png'), allowed), + false + ); + assert.equal( + manager.isImageImportHostnameAllowed(new URL('file:///etc/passwd'), allowed), + false + ); + }); + }); }); From ffa58b3d607c544649cb3a260869253ddb31afde Mon Sep 17 00:00:00 2001 From: Tom Boutell Date: Wed, 13 May 2026 10:22:41 -0400 Subject: [PATCH 25/56] mergeback (#5409) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Latest merge prerelease 2026 05 (#5404) * Fix choices IDs (#5379) * Fix choices IDs * Cleanup dots from ID values * Update changelog Co-authored-by: Robert Means --------- Co-authored-by: Robert Means (cherry picked from commit 9f458b5d9544d33239a0971b0050fff8bffe028a) * Bump CLI dependencies (#5383) (cherry picked from commit a5e1a4a35d01e09cbfd2c503370c95a8c2490867) * Native browser shortcuts work again (#5384) (cherry picked from commit b9b32bdd446add927c75d85e44d0a26814ba62b0) * Pro 8838 charset (#5385) * Removes encoding option and comments hardcoded encoding meta * add changeset * Response to first comments (cherry picked from commit 08845c5f23d95ead552ab9743a878daf997793a1) * Log aposResponse errors (#5386) * log aposResponse errors * add changeset --------- Co-authored-by: Harouna Traoré (cherry picked from commit d45e27f6d66cd25fde0cf9c7199ee9d014fa1518) * merge back the thanks (#5388) (cherry picked from commit f3501f4de0448d4d0f8a01a1db4dc2fcf52c6218) * ignore inline table array as draggable ui for windows (#5392) (cherry picked from commit b360b05e8d4e479a36fcae45a5a6dbf820b8fba3) * Layout focus orchestration (#5393) (cherry picked from commit 77a2968206b0208883ff45de37feb43ba0b5d5b1) * Bump dependencies (#5398) * Bump dependencies * Fix missing test await resulting in random failures (cherry picked from commit 008417fa9814dba4899b53b9394778dbd46ccf3c) * PRO-9467: remove defunct and nonpublic oembed providers and improve developer control to ensure security in the future (no risk exists today) (#5400) * PRO-9467: remove defunct and nonpublic oembed providers and improve developer control to ensure security in the future (no risk exists today) * allow newer twitter domain * infogr.am still around * facebook no longer does oembed (cherry picked from commit e9b3bac692315414da47fe05202e8854dd31d7ee) * Layout editable gap (#5397) (cherry picked from commit bc8f7bed381d4462a082976dd92b8d7f28180a95) * a11y fixes (#5401) (cherry picked from commit 2e2f3b4b46301fc81c46200f837e794b247a9aef) * clarifications (#5403) (cherry picked from commit 13f2c69f219a47f82972d8fa9036deddd84223dc) --------- Co-authored-by: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Co-authored-by: Robert Means Co-authored-by: haroun <1765606+haroun@users.noreply.github.com> Co-authored-by: Harouna Traoré Co-authored-by: Stuart Romanek * Latest security merge (#5407) * Merge commit from fork * Fixed arbitrary image URL import vulnerability in rich text impport API * additional reporter * Merge commit from fork * Merge commit from fork * Merge commit from fork * secure the link URL field of image widgets * credit * Merge commit from fork * fix xmp tag vulnerability * thanks * Merge commit from fork * Security: a malicious full name containing HTML was executed as HTML in the tooltip displayed with an "i" icon next to the title of the current page, creating an XSS attack risk versus other users. Since most projects permit users to change their full name (the "title" property), All projects with multiple users should be updated promptly to close this vulnerability. * changeset * release only (changelogs formatted) (#5408) --------- Co-authored-by: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Co-authored-by: Robert Means Co-authored-by: haroun <1765606+haroun@users.noreply.github.com> Co-authored-by: Harouna Traoré Co-authored-by: Stuart Romanek --- packages/apostrophe-astro/CHANGELOG.md | 7 +++++ packages/apostrophe-astro/package.json | 2 +- packages/apostrophe/CHANGELOG.md | 34 +++++++++++++++++++++++ packages/apostrophe/package.json | 2 +- packages/cli/CHANGELOG.md | 8 ++++++ packages/cli/package.json | 2 +- packages/launder/CHANGELOG.md | 38 +++++++++++++++----------- packages/launder/package.json | 2 +- packages/sanitize-html/CHANGELOG.md | 10 +++++++ packages/sanitize-html/package.json | 2 +- 10 files changed, 86 insertions(+), 21 deletions(-) diff --git a/packages/apostrophe-astro/CHANGELOG.md b/packages/apostrophe-astro/CHANGELOG.md index fbac106e56..95757b3190 100644 --- a/packages/apostrophe-astro/CHANGELOG.md +++ b/packages/apostrophe-astro/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 1.12.0 + +### Adds + +- Editors can now control the layout-widget gap through the styles system, both site-wide via a global `layoutGap` preset and per widget via a `gap` styles field. New Layout widget option `className` allows for additional CSS class names to be added to the widget Grid container. +- Log aposResponse errors server side in the Astro process. Thanks to [Harouna Traore](https://github.com/haroun). + ## 1.11.0 (2026-04-15) ### Adds diff --git a/packages/apostrophe-astro/package.json b/packages/apostrophe-astro/package.json index 1018711f74..eb8545a881 100644 --- a/packages/apostrophe-astro/package.json +++ b/packages/apostrophe-astro/package.json @@ -1,6 +1,6 @@ { "name": "@apostrophecms/apostrophe-astro", - "version": "1.11.0", + "version": "1.12.0", "type": "module", "description": "Apostrophe integration for Astro", "repository": { diff --git a/packages/apostrophe/CHANGELOG.md b/packages/apostrophe/CHANGELOG.md index 6443d761f2..982e6952c0 100644 --- a/packages/apostrophe/CHANGELOG.md +++ b/packages/apostrophe/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## 4.30.0 + +### Adds + +- Layout widget gap is now controllable through the styles system, both site-wide via a global `layoutGap` preset and per widget via a `gap` styles field. A new `className` option allows additional CSS classes to be added to the widget grid container. + +### Fixes + +- Fixed layout widget not regaining full focus when switching back to Edit content mode. +- Fixed illegal HTML `id` attribute values generated by the admin UI. +- Fixed orderable table array items dragging the entire floating window. +- Fixed keyboard shortcuts for widget operations (copy, cut, paste, duplicate, remove) blocking the browser's native clipboard behavior when no widget was focused. Previously, selecting and copying text while logged in was intercepted unconditionally by the admin UI. +- Removed duplicate `` tag from `outerLayoutBase.html` and standardized charset to `utf-8`. +- Updated `apostrophe` and `oembetter` to remove oembed services that no longer support public access, eliminating them as a potential future XSS vector. New `minimumAllowlist` and `minimumEndpoints` options on `@apostrophecms/oembed` allow developers to prune the list further. + +### Security + +- **Password reset base URL requirement:** The password reset feature now refuses to operate unless `baseUrl` or `APOS_BASE_URL` is set, preventing a vulnerability where ApostropheCMS could be convinced to send emails with links to attacker-controlled sites. Only affects projects with `passwordReset: true` on the login module. Thanks to [SPIDY](https://github.com/Mujahidkhan525) for reporting. +- **XSS via full name field:** A malicious full name containing HTML was executed in the page title tooltip in the admin bar, posing an XSS risk to other users. All multi-user projects should update promptly. Thanks to [Muhammad Uwais](https://github.com/MuhammadUwais) for reporting. +- **XSS via image widget link URL:** Users with editing privileges could trigger arbitrary JavaScript via a `javascript:` URL in the image widget's link URL field. A migration is included to strip any such URLs already in the database. Thanks to [Muhammad Uwais](https://github.com/MuhammadUwais) for reporting. +- **SSRF via rich text HTML import:** The rich text widget's HTML import feature no longer fetches images from arbitrary hosts, which could be used to probe internal networks or exfiltrate internal images. Configure `imageImportAllowedHostnames` on `@apostrophecms/rich-text-widget` to opt in. Thanks to [Yiğit Şengezer](https://github.com/yigitsengezer) and [Sainithin0309](https://github.com/Sainithin0309) for reporting. +- **the xmp tag could be used to pass forbidden markup through sanitize-html**, even when xmp itself. This was fixed in `sanitize-html` and the dependency was bumped. Thanks to [Vincenzo Turturro](https://github.com/sushi-gif) for reporting the vulnerability. +- **the `linkHref` field of image widgets was an XSS vulnerability** because it did not use the `url` field type. This means that a user with editing privileges could potentially carry out XSS. In addition, we have updated the `launder` module to sanitize URLs more robustly for the `url` field type, and bumped that dependency. Also, a database migration is included to clean any XSS attacks that could be present in existing links. Thanks to [Muhammad Uwais](https://github.com/MuhammadUwais) for reporting the issue. + +### Accessibility + +- Corrected ARIA semantics on the top admin navigation bar. +- Improved the document context title (admin bar middle group) and the underlying `AposContextMenu` machinery. +- Improved the locale switcher (`AposLocalePicker`). +- The Recently Edited Documents tray icon now exposes its action via `aria-label`. +- Fixed `.apos-sr-only` so screen-reader-only content is correctly exposed to the accessibility tree. +- Icon-only context-utility buttons in the admin bar tray (e.g. the global settings cog) now expose their action via `aria-label`. + + ## 4.29.0 (2026-04-15) ### Adds diff --git a/packages/apostrophe/package.json b/packages/apostrophe/package.json index f33c12137c..850b2c9d12 100644 --- a/packages/apostrophe/package.json +++ b/packages/apostrophe/package.json @@ -1,6 +1,6 @@ { "name": "apostrophe", - "version": "4.29.0", + "version": "4.30.0", "description": "The Apostrophe Content Management System.", "main": "index.js", "scripts": { diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index e6b0c80964..517d506fb7 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 3.6.1 + +### Security + +- Bump and clean up dependencies. This closes vulnerabilities in `uuid` and `fast-xml-parser` although they were not used in a sensitive or vulnerable way within ApostropheCMS. This also closes a vulnerability in `shelljs` which ould only be exploited if the developer could be convinced to enter malicious commands as part of their CLI input. +- Passwords and starter kit URLs containing intentionally malicious punctuation cannot be used to run arbitrary shell commands. Because the CLI is only used by developers, this would always have been an "own goal" situation, however this does make the CLI more robust for scripted use. Thanks to [Nitro13urn](https://github.com/VadlaReddySai) for reporting the issue. + ## 3.6.0 (2026-02-18) ### Adds @@ -11,6 +18,7 @@ - The `add` command now displays an error when run inside a hybrid Astro project, as it is not currently supported in that context. - The default starter kit is now `public-demo` + - Validates the `shortName` argument in the `create` command to only allow letters, numbers, hyphens, and underscores, preventing potential command injection. ## 3.5.0 (2025-01-07) diff --git a/packages/cli/package.json b/packages/cli/package.json index c4f9221650..fbad8f1f39 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@apostrophecms/cli", - "version": "3.6.0", + "version": "3.6.1", "description": "Commandline generator and configurator for Apostrophe CMS", "main": "bin/apostrophe", "scripts": { diff --git a/packages/launder/CHANGELOG.md b/packages/launder/CHANGELOG.md index 02c1338be2..05384c6951 100644 --- a/packages/launder/CHANGELOG.md +++ b/packages/launder/CHANGELOG.md @@ -1,67 +1,73 @@ # Changelog +## 1.7.1 + +### Security + +- `launder` now uses and exports the best available naughtyHref function for detecting malicious URLs. sanitize-html now depends on it, and apostrophe now uses type: 'url' for the link URL field of image widgets, which leverages it. Prior to this fix, it was possible for any user with editing privileges, including a contributor, to trigger arbitrary JavaScript via a javascript: URL in the link URL field of an image widget. A migration has been included to strip any such malicious URLs already present in the database. All users of apostrophe are encouraged to upgrade to get this security fix. Thanks to [Muhammad Uwais](https://github.com/MuhammadUwais) for reporting the issue. + ## 1.7.0 - 2023-05-03 -* Modernize dependencies to correct installation warnings, and eliminate use of `var`. +- Modernize dependencies to correct installation warnings, and eliminate use of `var`. ## 1.6.0 - 2023-03-06 -* `sms:` urls are now accepted. Thanks to [Ruben](https://github.com/reubendsouza) for this change. +- `sms:` urls are now accepted. Thanks to [Ruben](https://github.com/reubendsouza) for this change. ## 1.5.1 - 2021-10-28 -* The `date` method now correctly returns `null` if the date argument is unparseable and the `def` parameter is explicitly `null`. As was always documented, a default of `undefined` still returns the current date. +- The `date` method now correctly returns `null` if the date argument is unparseable and the `def` parameter is explicitly `null`. As was always documented, a default of `undefined` still returns the current date. ## 1.5.0 -* The `url` method now accepts a third argument, `httpsFix`. If it is `true` and the URL passed in has no protocol, the URL will be prepended with `https://` rather than `http://`. +- The `url` method now accepts a third argument, `httpsFix`. If it is `true` and the URL passed in has no protocol, the URL will be prepended with `https://` rather than `http://`. ## 1.4.0 -* `tel:` urls are now accepted. +- `tel:` urls are now accepted. ## 1.3.0 -* `booleanOrNull` accepts the string `'null'` as a synonym for `null`. Note that `'any'` was already accepted. `'null'` can be an attractive choice when the user will not see it in the query string and conflict with other uses of `'any'` is a concern. +- `booleanOrNull` accepts the string `'null'` as a synonym for `null`. Note that `'any'` was already accepted. `'null'` can be an attractive choice when the user will not see it in the query string and conflict with other uses of `'any'` is a concern. ## 1.2.0 -* `idRegExp` option may be passed to change the rules for `launder.id`. +- `idRegExp` option may be passed to change the rules for `launder.id`. ## 1.1.2 -* linting. +- linting. ## 1.1.1 -* for improved bc, `launder.select` does not crash if some of the choices given to `select` are null or undefined. Although this is a developer error rather than a sanitization issue, versions prior to 1.1.0 did tolerate this situation, so 1.1.1 does so as well. Thanks to Anthony Tarlao for his code contributions, and to Michelin for making this fix possible via [Apostrophe Enterprise Support](https://apostrophecms.com/support/enterprise-support). +- for improved bc, `launder.select` does not crash if some of the choices given to `select` are null or undefined. Although this is a developer error rather than a sanitization issue, versions prior to 1.1.0 did tolerate this situation, so 1.1.1 does so as well. Thanks to Anthony Tarlao for his code contributions, and to Michelin for making this fix possible via [Apostrophe Enterprise Support](https://apostrophecms.com/support/enterprise-support). ## 1.1.0 -* `launder.select` now handles numeric values for choices gracefully. Specifically, if the value passed in is a string, it will be validated as a match for a choice that is a number, as long as they have the same string representation, and the number (not the string) will be returned. Previously there was no match in this situation. +- `launder.select` now handles numeric values for choices gracefully. Specifically, if the value passed in is a string, it will be validated as a match for a choice that is a number, as long as they have the same string representation, and the number (not the string) will be returned. Previously there was no match in this situation. Thanks to Michelin for making this possible via [Apostrophe Enterprise Support](https://apostrophecms.com/support/enterprise-support). ## 1.0.1 -* `launder.time` will now also accept a `.` (dot) as the separator (until now only `:` colon was recognized). Thanks to Lars Houmark. +- `launder.time` will now also accept a `.` (dot) as the separator (until now only `:` colon was recognized). Thanks to Lars Houmark. ## 1.0.0 -* switched to a maintained, secure fork of lodash 3, declared 1.0.0 as this has been a stable part of Apostrophe for years. +- switched to a maintained, secure fork of lodash 3, declared 1.0.0 as this has been a stable part of Apostrophe for years. ## 0.1.3 -* `launder.booleanOrNull` broken out from `launder.addBooleanFilterCriteria` so that you can get the tri-state value without modifying a criteria object. +- `launder.booleanOrNull` broken out from `launder.addBooleanFilterCriteria` so that you can get the tri-state value without modifying a criteria object. ## 0.1.2 -* `launder.tags` also accepts a comma-separated string. +- `launder.tags` also accepts a comma-separated string. ## 0.1.1 -* removed never-used and undocumented `parseTime` method. +- removed never-used and undocumented `parseTime` method. ## 0.1.0 -* initial release. Based on stable code recently refactored from Apostrophe 0.5.x. +- initial release. Based on stable code recently refactored from Apostrophe 0.5.x. diff --git a/packages/launder/package.json b/packages/launder/package.json index aa01b14a36..de42a61358 100644 --- a/packages/launder/package.json +++ b/packages/launder/package.json @@ -1,6 +1,6 @@ { "name": "launder", - "version": "1.7.0", + "version": "1.7.1", "description": "A sanitize module for the people. Built for ApostropheCMS.", "main": "index.js", "scripts": { diff --git a/packages/sanitize-html/CHANGELOG.md b/packages/sanitize-html/CHANGELOG.md index 9661f24184..b5663fa2b9 100644 --- a/packages/sanitize-html/CHANGELOG.md +++ b/packages/sanitize-html/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 2.17.4 + +### Changes + +- `sanitize-html` and `launder` now share a single implementation of `naughtyHref`, based on that which previously existed in `sanitize-html`. + +### Security + +- Security vulnerability: the xmp tag could be used to pass forbidden markup through sanitize-html, even when xmp itself is not explicitly allowed All users of sanitize-html should update immediately. Thanks to [Vincenzo Turturro](https://github.com/sushi-gif) for reporting the vulnerability. + ## 2.17.3 (2026-04-15) ### Security diff --git a/packages/sanitize-html/package.json b/packages/sanitize-html/package.json index fe401c0764..a73117759d 100644 --- a/packages/sanitize-html/package.json +++ b/packages/sanitize-html/package.json @@ -1,6 +1,6 @@ { "name": "sanitize-html", - "version": "2.17.3", + "version": "2.17.4", "description": "Clean up user-submitted HTML, preserving allowlisted elements and allowlisted attributes on a per-element basis", "sideEffects": false, "main": "index.js", From e52e1df071f5e1fc83bdbe169e63ff6241717b85 Mon Sep 17 00:00:00 2001 From: Tom Boutell Date: Wed, 13 May 2026 15:16:29 -0400 Subject: [PATCH 26/56] mergeback (#5414) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Latest merge prerelease 2026 05 (#5404) * Fix choices IDs (#5379) * Fix choices IDs * Cleanup dots from ID values * Update changelog Co-authored-by: Robert Means --------- Co-authored-by: Robert Means (cherry picked from commit 9f458b5d9544d33239a0971b0050fff8bffe028a) * Bump CLI dependencies (#5383) (cherry picked from commit a5e1a4a35d01e09cbfd2c503370c95a8c2490867) * Native browser shortcuts work again (#5384) (cherry picked from commit b9b32bdd446add927c75d85e44d0a26814ba62b0) * Pro 8838 charset (#5385) * Removes encoding option and comments hardcoded encoding meta * add changeset * Response to first comments (cherry picked from commit 08845c5f23d95ead552ab9743a878daf997793a1) * Log aposResponse errors (#5386) * log aposResponse errors * add changeset --------- Co-authored-by: Harouna Traoré (cherry picked from commit d45e27f6d66cd25fde0cf9c7199ee9d014fa1518) * merge back the thanks (#5388) (cherry picked from commit f3501f4de0448d4d0f8a01a1db4dc2fcf52c6218) * ignore inline table array as draggable ui for windows (#5392) (cherry picked from commit b360b05e8d4e479a36fcae45a5a6dbf820b8fba3) * Layout focus orchestration (#5393) (cherry picked from commit 77a2968206b0208883ff45de37feb43ba0b5d5b1) * Bump dependencies (#5398) * Bump dependencies * Fix missing test await resulting in random failures (cherry picked from commit 008417fa9814dba4899b53b9394778dbd46ccf3c) * PRO-9467: remove defunct and nonpublic oembed providers and improve developer control to ensure security in the future (no risk exists today) (#5400) * PRO-9467: remove defunct and nonpublic oembed providers and improve developer control to ensure security in the future (no risk exists today) * allow newer twitter domain * infogr.am still around * facebook no longer does oembed (cherry picked from commit e9b3bac692315414da47fe05202e8854dd31d7ee) * Layout editable gap (#5397) (cherry picked from commit bc8f7bed381d4462a082976dd92b8d7f28180a95) * a11y fixes (#5401) (cherry picked from commit 2e2f3b4b46301fc81c46200f837e794b247a9aef) * clarifications (#5403) (cherry picked from commit 13f2c69f219a47f82972d8fa9036deddd84223dc) --------- Co-authored-by: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Co-authored-by: Robert Means Co-authored-by: haroun <1765606+haroun@users.noreply.github.com> Co-authored-by: Harouna Traoré Co-authored-by: Stuart Romanek * Latest security merge (#5407) * Merge commit from fork * Fixed arbitrary image URL import vulnerability in rich text impport API * additional reporter * Merge commit from fork * Merge commit from fork * Merge commit from fork * secure the link URL field of image widgets * credit * Merge commit from fork * fix xmp tag vulnerability * thanks * Merge commit from fork * Security: a malicious full name containing HTML was executed as HTML in the tooltip displayed with an "i" icon next to the title of the current page, creating an XSS attack risk versus other users. Since most projects permit users to change their full name (the "title" property), All projects with multiple users should be updated promptly to close this vulnerability. * changeset * release only (changelogs formatted) (#5408) * allow oembetter to be released (#5412) * release oembetter 1.2.0 (#5413) * release oembetter 1.2.0 * left commit --------- Co-authored-by: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Co-authored-by: Robert Means Co-authored-by: haroun <1765606+haroun@users.noreply.github.com> Co-authored-by: Harouna Traoré Co-authored-by: Stuart Romanek --- packages/oembetter/CHANGELOG.md | 32 ++++++++++++++++++++++++++++++++ packages/oembetter/package.json | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/oembetter/CHANGELOG.md b/packages/oembetter/CHANGELOG.md index 10c9aca99a..652f3ffec9 100644 --- a/packages/oembetter/CHANGELOG.md +++ b/packages/oembetter/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 1.2.0 + +### Changes + +- remove obsolete services and those which no longer support anonymous oembed from the suggestedAllowlist and suggestedEndpoints, to ensure they do not become a security risk in the future +- update the fast-xml-parser dependency + +### Adds + +- add x.com endpoint + ## 1.1.4 (2024-08-07) - Followup to 1.1.3: also hardcode the YouTube oembed endpoint for sharing URLs (`youtu.be`). @@ -33,67 +44,88 @@ - Updates `cheerio` to the 1.0.0-rc version series to address a security vulnerability. ## 1.0.0 + renamed the `whitelist` and `suggestedWhitelist` properties to `allowlist` and `suggestedAllowlist`, respectively. Also introduced support for `options.headers`. ## 0.1.23 + workaround for YouTube bug in which video pages contain `http:` recommendations for oembed URLs, but an `http:` request is rejected with a 403 error. Force `https:` for YouTube. ## 0.1.22 + fixed URL parsing bugs impacting use of preconfigured endpoints that already contain some query string parameters. ## 0.1.21 + Updated links and information in the README. ## 0.1.20 + fixed a nuisance error that was appearing when Facebook was present but `window` was not the default object. ## 0.1.19 + unnecessary Facebook API logic was running on non-Facebook embeds due to a syntax mistake in 0.1.17. ## 0.1.18 + report HTTP errors properly rather than attempting to parse a nonexistent JSON body. Also, always try/catch when parsing JSON and report the exception as the callback error if necessary. ## 0.1.17 + Facebook oembed filter works regardless of whether Facebook's API has been initialized yet or not. ## 0.1.16 + Built in filter that handles Facebook oembed responses. ## 0.1.15 + allowlisted `facebook.com`, which has extensive oembed these days. ## 0.1.14 + bumped `cheerio` dependency to fix deprecation warnings. No behavior changes. ## 0.1.13 + relative URLs work with discovery. Thanks to Alejandro Torrado. ## 0.1.12 + (unchanged, npm publishing issue) ## 0.1.11 + don't crash when evaluating allowlists if `parsed.hostname` somehow manages not to be set. ## 0.1.10 + user agent string to please Facebook. Thanks to `equinox7`. ## 0.1.9 + the new `endpoints` option allows you to configure custom oembed API endpoints for services that don't advertise an endpoint or advertise it incorrectly. ## 0.1.7-0.1.8 + support SoundCloud. Added it to the suggested allowlist and added tolerance for their incorrect JSON content type. ## 0.1.6 + security improvement: reject all URLs that are not `http:` or `https:` completely, right up front. This means you don't have to protect against these obvious hacks in your `before` and `after` handlers. ## 0.1.5 + packaging issues, no changes. ## 0.1.4 + if the URL leads to a page with no oembed metadata, look for a `link rel="canonical"` tag and try that URL instead. Don't pursue this more than one step. Also, specify a user agent so that certain hosts don't give us watered-down HTML. ## 0.1.3 + added `youtu.be` to the suggested allowlist. diff --git a/packages/oembetter/package.json b/packages/oembetter/package.json index 63ee9c08c2..a92b916db9 100644 --- a/packages/oembetter/package.json +++ b/packages/oembetter/package.json @@ -1,6 +1,6 @@ { "name": "oembetter", - "version": "1.1.4", + "version": "1.2.0", "description": "A modern oembed client. Allows you to register filters to improve or supply oembed support for sites that don't normally have it. You can also supply a allowlist of services you trust to prevent XSS attacks.", "main": "index.js", "scripts": { From cf1a63984acd76437608969e82e1b4e095dca0e6 Mon Sep 17 00:00:00 2001 From: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Date: Tue, 19 May 2026 11:14:01 +0300 Subject: [PATCH 27/56] Fix focus trap on the last element in a modal (#5406) * Fix focus trap on the last element in a modal * Fix trap escaping edge cases --- .changeset/moody-poems-speak.md | 7 ++ .../modal/ui/apos/components/AposModal.vue | 72 +++++++++++++------ 2 files changed, 56 insertions(+), 23 deletions(-) create mode 100644 .changeset/moody-poems-speak.md diff --git a/.changeset/moody-poems-speak.md b/.changeset/moody-poems-speak.md new file mode 100644 index 0000000000..8582885ec2 --- /dev/null +++ b/.changeset/moody-poems-speak.md @@ -0,0 +1,7 @@ +--- +"apostrophe": minor +--- + +Fixed an issue where using the Tab key to navigate within modals could incorrectly jump focus to a wrong element instead of the next input field. + +Fixed Tab navigation escaping out of modals when the form contained hidden sections or elements that became disabled after editing. diff --git a/packages/apostrophe/modules/@apostrophecms/modal/ui/apos/components/AposModal.vue b/packages/apostrophe/modules/@apostrophecms/modal/ui/apos/components/AposModal.vue index a83770ff35..055434d48e 100644 --- a/packages/apostrophe/modules/@apostrophecms/modal/ui/apos/components/AposModal.vue +++ b/packages/apostrophe/modules/@apostrophecms/modal/ui/apos/components/AposModal.vue @@ -16,7 +16,7 @@ :data-apos-graph-key="props.graphKey || undefined" tabindex="0" @focus.capture="captureFocus" - @keyup.tab="onKeyup" + @keydown.tab="onKeydownTab" @keyup.esc="onKeyup" > @@ -216,6 +216,22 @@ const nonDraggableElements = [ '.apos-input-array-inline-table' ]; +// Selector for focusable elements inside the modal. Used both at trap setup +// and on every Tab keydown to refresh the cycle list, so that elements that +// became disabled/hidden (e.g. Save when validation fails) or newly visible +// are reflected. +const focusableSelector = [ + '[tabindex]', + '[href]', + 'input', + 'select', + 'textarea', + 'button', + '[data-apos-focus-priority]' +] + .map(s => `${s}:not([tabindex="-1"]):not([disabled]):not([type="hidden"]):not([aria-hidden]):not(.apos-sr-only)`) + .join(', '); + const resizeSides = [ { edge: 'top', @@ -476,13 +492,30 @@ onUnmounted(() => { } }); -function onKeyup(event) { +// Handle Tab on keydown — before the browser moves focus. +// Handling Tab on keyup is too late: the browser has already moved focus, +// so the cycling logic sees the wrong activeElement. +// +// We also recompute the focusable list here on every Tab, scoped to +// modalEl, instead of relying on the snapshot taken at trapFocus() time. +function onKeydownTab(event) { + if (!shouldTrapFocus.value) { + return; + } if (!store.isOnTop(modalEl.value)) { return; } + if (event.target?.nodeName?.toLowerCase() === 'textarea') { + return; + } + const elements = getFocusableElements(modalEl.value); + // Keep the store snapshot consistent for other consumers. + store.updateModalData(props.modalData.id, { elementsToFocus: elements }); + cycleElementsToFocus(event, elements); +} - if (event.key === 'Tab') { - cycleElementsToFocus(event, props.modalData.elementsToFocus); +function onKeyup(event) { + if (!store.isOnTop(modalEl.value)) { return; } @@ -517,23 +550,20 @@ function captureFocus(e) { store.updateModalData(props.modalData.id, { focusedElement: e.target }); } +function getFocusableElements(rootEl) { + if (!rootEl) { + return []; + } + return [ ...rootEl.querySelectorAll(focusableSelector) ] + // a cheap "visible and in the DOM" condition, + // false positive expected for position: fixed and + // visually hidden elements (visible: hidden, opacity: 0, etc.) + .filter(el => el.offsetParent !== null); +} + async function trapFocus() { if (modalEl?.value) { - const elementSelectors = [ - '[tabindex]', - '[href]', - 'input', - 'select', - 'textarea', - 'button', - '[data-apos-focus-priority]' - ]; - - const selector = elementSelectors - .map(addExcludingAttributes) - .join(', '); - - const elementsToFocus = [ ...modalEl.value.querySelectorAll(selector) ]; + const elementsToFocus = getFocusableElements(modalEl.value); store.updateModalData(props.modalData.id, { elementsToFocus }); @@ -556,10 +586,6 @@ async function trapFocus() { await nextTick(); focusElement(props.modalData.focusedElement, firstElementToFocus); } - - function addExcludingAttributes(element) { - return `${element}:not([tabindex="-1"]):not([disabled]):not([type="hidden"]):not([aria-hidden]):not(.apos-sr-only)`; - } } function close() { From 167c2b5c1d63ba4b01726a2295b92e3f8dd94a31 Mon Sep 17 00:00:00 2001 From: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Date: Tue, 19 May 2026 11:31:58 +0300 Subject: [PATCH 28/56] Fix import-export noise (#5399) * remove noise, switch to utils debug * Fix tests * Introduce debug option * changelog * Fix test sorting issue --- .changeset/shy-cougars-switch.md | 5 +++++ packages/import-export/index.js | 5 ++++- packages/import-export/lib/formats/csv.js | 2 -- packages/import-export/lib/formats/gzip.js | 10 ---------- packages/import-export/lib/handlers.js | 4 ++-- packages/import-export/lib/methods/export.js | 8 ++++---- packages/import-export/lib/methods/import.js | 2 +- packages/import-export/lib/methods/index.js | 15 ++++++++++++++- packages/import-export/package.json | 2 +- packages/import-export/test/import-page.js | 16 ++++++++++++++++ packages/import-export/test/index.js | 14 ++++++++++++-- 11 files changed, 59 insertions(+), 24 deletions(-) create mode 100644 .changeset/shy-cougars-switch.md diff --git a/.changeset/shy-cougars-switch.md b/.changeset/shy-cougars-switch.md new file mode 100644 index 0000000000..498dfa5775 --- /dev/null +++ b/.changeset/shy-cougars-switch.md @@ -0,0 +1,5 @@ +--- +"@apostrophecms/import-export": patch +--- + +Debug logging is now disabled by default and can be enabled by setting the `debug: true` option on the `@apostrophecms/import-export` module, or by setting the `APOS_DEBUG_IMPORT_EXPORT=1` environment variable. diff --git a/packages/import-export/index.js b/packages/import-export/index.js index 8418461470..e58d0baca3 100644 --- a/packages/import-export/index.js +++ b/packages/import-export/index.js @@ -20,7 +20,10 @@ module.exports = { browser: true }, preventUpdateAssets: false, - importDraftsOnlyDefault: false + importDraftsOnlyDefault: false, + // When true (or when the `APOS_DEBUG_IMPORT_EXPORT='1'` env var is set), + // verbose internal progress is logged. + debug: false }, init(self) { self.formats = { diff --git a/packages/import-export/lib/formats/csv.js b/packages/import-export/lib/formats/csv.js index caa6779739..9b776df946 100644 --- a/packages/import-export/lib/formats/csv.js +++ b/packages/import-export/lib/formats/csv.js @@ -41,7 +41,6 @@ module.exports = { reader.on('error', reject); parser.on('error', reject); parser.on('end', () => { - console.info(`[csv] docs read from ${filepath}`); resolve({ docs }); }); }); @@ -74,7 +73,6 @@ module.exports = { stringifier.on('error', reject); writer.on('error', reject); writer.on('finish', () => { - console.info(`[csv] export file written to ${filepath}`); resolve(); }); }); diff --git a/packages/import-export/lib/formats/gzip.js b/packages/import-export/lib/formats/gzip.js index 763874fb36..a70e53c337 100644 --- a/packages/import-export/lib/formats/gzip.js +++ b/packages/import-export/lib/formats/gzip.js @@ -24,10 +24,8 @@ module.exports = { if (filepath.endsWith(this.allowedExtension)) { exportPath = filepath.replace(this.allowedExtension, ''); - console.info(`[gzip] extracting ${filepath} into ${exportPath}`); await extract(filepath, exportPath); - console.info(`[gzip] removing ${filepath}`); await remove(filepath); } @@ -35,10 +33,6 @@ module.exports = { const attachmentsPath = path.join(exportPath, 'aposAttachments.json'); const attachmentFilesPath = path.join(exportPath, 'attachments'); - console.info(`[gzip] reading docs from ${docsPath}`); - console.info(`[gzip] reading attachments from ${attachmentsPath}`); - console.info(`[gzip] reading attachment files from ${attachmentFilesPath}`); - const docs = await fsp.readFile(docsPath); const attachments = await fsp.readFile(attachmentsPath); @@ -92,12 +86,10 @@ module.exports = { pack.on('error', reject); writeStream.on('finish', () => { - console.info(`[gzip] export file written to ${filepath}`); resolve(result); }); for (const [ filename, content ] of Object.entries(data.json || {})) { - console.info(`[gzip] adding ${filename} to the tarball`); addTarEntry(pack, { name: filename }, content).catch(reject); } @@ -107,8 +99,6 @@ module.exports = { }) .then(() => { processAttachments(data.attachments, async (attachmentPath, name, size) => { - console.info(`[gzip] adding attachments/${name} to the tarball`); - const readStream = fs.createReadStream(attachmentPath); const entryStream = pack.entry({ name: `attachments/${name}`, diff --git a/packages/import-export/lib/handlers.js b/packages/import-export/lib/handlers.js index b262ae48ed..2083c31bad 100644 --- a/packages/import-export/lib/handlers.js +++ b/packages/import-export/lib/handlers.js @@ -73,7 +73,7 @@ module.exports = self => { if (!ids.length) { return; } - self.apos.util.debug(`Clearing ${ids.length} timer(s)`); + self.debug(`Clearing ${ids.length} timer(s)`); for (const key of ids) { const entry = self.timeoutIds[key]; delete self.timeoutIds[key]; @@ -85,7 +85,7 @@ module.exports = self => { await entry.handler(); } } - self.apos.util.debug('Timer(s) cleared'); + self.debug('Timer(s) cleared'); } } }; diff --git a/packages/import-export/lib/methods/export.js b/packages/import-export/lib/methods/export.js index ae22d3fe7e..ac107ca6e7 100644 --- a/packages/import-export/lib/methods/export.js +++ b/packages/import-export/lib/methods/export.js @@ -510,7 +510,7 @@ module.exports = self => { const downloadPath = path.join('/exports', filename); const downloadUrl = `${self.apos.attachment.uploadfs.getUrl()}${downloadPath}`; const copyIn = util.promisify(self.apos.attachment.uploadfs.copyIn); - console.info(`[export] copying ${filepath} to ${self.apos.rootDir}/public/uploads${downloadPath}`); + self.debug(`[export] copying ${filepath} to ${self.apos.rootDir}/public/uploads${downloadPath}`); try { await copyIn(filepath, downloadPath); } catch (error) { @@ -565,7 +565,7 @@ module.exports = self => { async function processAttachment([ name, url ]) { const temp = self.apos.attachment.uploadfs.getTempPath() + '/' + self.apos.util.generateId(); - console.info(`[export] processing attachment ${name} temporarily stored in ${temp}`); + self.debug(`[export] processing attachment ${name} temporarily stored in ${temp}`); try { await copyOut(url, temp); const { size } = await fsp.stat(temp); @@ -593,9 +593,9 @@ module.exports = self => { removeFromUploadFs(downloadPath, expiration) { const ms = expiration || 1000 * 60 * 10; const id = uniqueId(); - console.info(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs in ${ms / 1000 / 60} minutes`); + self.debug(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs in ${ms / 1000 / 60} minutes`); const handler = () => { - console.info(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs`); + self.debug(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs`); delete self.timeoutIds[id]; return new Promise((resolve, _reject) => { self.apos.attachment.uploadfs.remove(downloadPath, error => { diff --git a/packages/import-export/lib/methods/import.js b/packages/import-export/lib/methods/import.js index 4d8226226e..747f20001a 100644 --- a/packages/import-export/lib/methods/import.js +++ b/packages/import-export/lib/methods/import.js @@ -1272,7 +1272,7 @@ module.exports = self => { }, async cleanExport(req) { - console.info('[import] cleaning export...'); + self.debug('[import] cleaning export...'); const exportPath = await self.getExportPath( self.apos.launder.string(req.body.exportId) ); diff --git a/packages/import-export/lib/methods/index.js b/packages/import-export/lib/methods/index.js index 42cf98ce9b..1d1613270d 100644 --- a/packages/import-export/lib/methods/index.js +++ b/packages/import-export/lib/methods/index.js @@ -2,8 +2,21 @@ const fsp = require('node:fs/promises'); const importMethods = require('./import.js'); const exportMethods = require('./export.js'); +const noop = () => {}; + module.exports = self => { + const debugEnabled = self.options.debug === true || + process.env.APOS_DEBUG_IMPORT_EXPORT === '1'; + // Bind once at startup so each call site pays no per-call cost. + // No-op unless the module's `debug` option is `true` or the + // `APOS_DEBUG_IMPORT_EXPORT` env var is set. + const debug = debugEnabled + ? (...args) => self.apos.util.debug(...args) + : noop; + return { + debug, + registerFormats(formats = {}) { verifyFormats(formats); @@ -56,7 +69,7 @@ module.exports = self => { } else { await fsp.unlink(filepath); } - console.info(`removed: ${filepath}`); + self.debug(`removed: ${filepath}`); } catch (err) { console.trace(err); self.apos.util.error( diff --git a/packages/import-export/package.json b/packages/import-export/package.json index 6cce8ddd00..ab3b56118c 100644 --- a/packages/import-export/package.json +++ b/packages/import-export/package.json @@ -8,7 +8,7 @@ "stylelint": "stylelint ui/**/*.{scss,vue}", "lint": "npm run eslint && npm run stylelint", "mocha": "mocha", - "test": "(../../scripts/pro-if-needed && npm run lint && mocha --ignore=test/import-page.js && mocha test/import-page.js) ; ../../scripts/undo-pro-if-needed" + "test": "../../scripts/pro-if-needed && (npm run lint && mocha --ignore=test/import-page.js && mocha test/import-page.js); err=$?; ../../scripts/undo-pro-if-needed; exit $err" }, "repository": { "type": "git", diff --git a/packages/import-export/test/import-page.js b/packages/import-export/test/import-page.js index 889ba357e8..0792d338b1 100644 --- a/packages/import-export/test/import-page.js +++ b/packages/import-export/test/import-page.js @@ -365,6 +365,7 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'level 2 page 1', type: 'test-page', updatedAt: importedDocs.at(0).updatedAt, + importedAt: importedDocs.at(0).importedAt, updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', @@ -404,6 +405,7 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'level 2 page 1', type: 'test-page', updatedAt: importedDocs.at(1).updatedAt, + importedAt: importedDocs.at(1).importedAt, updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', @@ -443,6 +445,7 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'level 4 page 1', type: 'test-page', updatedAt: importedDocs.at(2).updatedAt, + importedAt: importedDocs.at(2).importedAt, updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', @@ -484,6 +487,7 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'level 4 page 1', type: 'test-page', updatedAt: importedDocs.at(3).updatedAt, + importedAt: importedDocs.at(3).importedAt, updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', @@ -614,6 +618,7 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'level 2 page 1', type: 'test-page', updatedAt: importedDocs.at(0).updatedAt, + importedAt: importedDocs.at(0).importedAt, updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', @@ -653,6 +658,7 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'level 2 page 1', type: 'test-page', updatedAt: importedDocs.at(1).updatedAt, + importedAt: importedDocs.at(1).importedAt, updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', @@ -692,6 +698,7 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'level 4 page 1', type: 'test-page', updatedAt: importedDocs.at(2).updatedAt, + importedAt: importedDocs.at(2).importedAt, updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', @@ -733,6 +740,7 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'level 4 page 1', type: 'test-page', updatedAt: importedDocs.at(3).updatedAt, + importedAt: importedDocs.at(3).importedAt, updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', @@ -985,6 +993,7 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'custom', type: 'custom-page', updatedAt: importedDocs.at(0).updatedAt, + importedAt: importedDocs.at(0).importedAt, updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', @@ -1024,6 +1033,10 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'custom', type: 'custom-page', updatedAt: importedDocs.at(1).updatedAt, + // No `importedAt`: this is a `:previous` mode revision (a snapshot + // of the prior published state demoted by re-publishing). The + // import flow only stamps the documents it actually inserts or + // updates (draft and published), not the audit copy. updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', @@ -1063,6 +1076,7 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'custom', type: 'custom-page', updatedAt: importedDocs.at(2).updatedAt, + importedAt: importedDocs.at(2).importedAt, updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', @@ -1102,6 +1116,7 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'custom level 3 page 1', type: 'test-page', updatedAt: importedDocs.at(3).updatedAt, + importedAt: importedDocs.at(3).importedAt, updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', @@ -1143,6 +1158,7 @@ describe('@apostrophecms/import-export:import-page', function () { titleSortified: 'custom level 3 page 1', type: 'test-page', updatedAt: importedDocs.at(4).updatedAt, + importedAt: importedDocs.at(4).importedAt, updatedBy: { _id: null, // TODO: should be my user id title: 'System Task', diff --git a/packages/import-export/test/index.js b/packages/import-export/test/index.js index d4fe8bd213..a8941a93b3 100644 --- a/packages/import-export/test/index.js +++ b/packages/import-export/test/index.js @@ -1791,7 +1791,7 @@ describe('@apostrophecms/import-export', function() { } ]; - assert.deepEqual(actual, expected); + assert.deepEqual(sortDocs(actual), sortDocs(expected)); }); it('should import and translate duplicated docs', async function () { @@ -2112,8 +2112,18 @@ describe('@apostrophecms/import-export', function() { } ]; - assert.deepEqual(actual, expected); + assert.deepEqual(sortDocs(actual), sortDocs(expected)); }); }); } }); + +// Sort an array of `{ type, aposLocale, title, modified }` projections by +// a deterministic composite key so assertions don't depend on insert order. +function sortDocs(docs) { + return [ ...docs ].sort((a, b) => { + const keyA = `${a.type}|${a.aposLocale}|${a.title}`; + const keyB = `${b.type}|${b.aposLocale}|${b.title}`; + return keyA < keyB ? -1 : keyA > keyB ? 1 : 0; + }); +} From 2191c8a6f31633418bafa67ed0ea156ed5605ba0 Mon Sep 17 00:00:00 2001 From: Miro Yovchev <2827783+myovchev@users.noreply.github.com> Date: Tue, 19 May 2026 12:51:08 +0300 Subject: [PATCH 29/56] A11y fixes part 3 (#5416) * Fix editor modal a11y issues * Fix manager a11y problems * Fix page manager a11y problems * fix media manager a11y issues * fix a11y issues in style editor and user settings * Fix login a11y issues * eliminate a modal issue * Remove bad aria in rich text * Fix wrong aria in layout * changelog * Fix totp a11y issues, doc context state safety * Fix uncaught error - popup blockers/tests --- .changeset/itchy-zoos-kick.md | 5 ++++ .../components/AposBreadcrumbOperations.vue | 1 - .../ui/apos/components/AposWidgetControls.vue | 1 - .../ui/apos/logic/AposDocContextMenu.js | 8 +++--- .../modules/@apostrophecms/i18n/i18n/en.json | 3 +++ .../components/AposMediaManagerEditor.vue | 4 +-- .../ui/apos/components/AposGridColumn.vue | 1 - .../ui/apos/components/AposGridManager.vue | 1 - .../login/ui/apos/components/TheAposLogin.vue | 12 +++++++-- .../ui/apos/components/TheAposLoginHeader.vue | 6 ++--- .../modal/ui/apos/components/AposModal.vue | 2 ++ .../ui/apos/components/AposModalTabs.vue | 7 +++++- .../@apostrophecms/piece-type/index.js | 3 ++- .../components/AposDocsManagerDisplay.vue | 9 +++++-- .../ui/apos/components/AposCellTitle.vue | 1 + .../components/AposRichTextWidgetEditor.vue | 25 ++++++++++++++++--- .../apos/components/AposInputDateAndTime.vue | 9 +++++-- .../ui/apos/components/AposInputSelect.vue | 1 + .../ui/apos/components/AposInputWrapper.vue | 2 +- .../schema/ui/apos/components/AposSubform.vue | 1 + .../schema/ui/apos/logic/AposSubform.js | 10 ++++++++ .../ui/apos/components/TheAposStyles.vue | 1 + .../ui/ui/apos/components/AposButtonGroup.vue | 2 +- .../ui/apos/components/AposCellLastEdited.vue | 2 +- .../ui/ui/apos/components/AposSelect.vue | 1 + .../ui/ui/apos/components/AposSlat.vue | 2 +- .../ui/apos/components/AposSubformPreview.vue | 2 +- .../ui/ui/apos/components/AposTreeHeader.vue | 2 +- .../ui/ui/apos/scss/global/_theme.scss | 1 + .../ui/apos/components/AposTotp.scss | 2 +- .../ui/apos/components/AposTotp.vue | 4 +-- 31 files changed, 98 insertions(+), 33 deletions(-) create mode 100644 .changeset/itchy-zoos-kick.md diff --git a/.changeset/itchy-zoos-kick.md b/.changeset/itchy-zoos-kick.md new file mode 100644 index 0000000000..2d0bf3e421 --- /dev/null +++ b/.changeset/itchy-zoos-kick.md @@ -0,0 +1,5 @@ +--- +"apostrophe": patch +--- + +Fix more admin UI a11y issues diff --git a/packages/apostrophe/modules/@apostrophecms/area/ui/apos/components/AposBreadcrumbOperations.vue b/packages/apostrophe/modules/@apostrophecms/area/ui/apos/components/AposBreadcrumbOperations.vue index 46f6c7851e..2ecb0e1b0e 100644 --- a/packages/apostrophe/modules/@apostrophecms/area/ui/apos/components/AposBreadcrumbOperations.vue +++ b/packages/apostrophe/modules/@apostrophecms/area/ui/apos/components/AposBreadcrumbOperations.vue @@ -119,7 +119,6 @@ export default { icon: 'plus-icon', type: 'group', modifiers: [ 'small', 'inline' ], - role: 'menuitem', class: 'apos-area-modify-controls__button', iconSize: 16, disableFocus: !this.isFocused diff --git a/packages/apostrophe/modules/@apostrophecms/area/ui/apos/components/AposWidgetControls.vue b/packages/apostrophe/modules/@apostrophecms/area/ui/apos/components/AposWidgetControls.vue index 8745ae5d2c..9c282b691b 100644 --- a/packages/apostrophe/modules/@apostrophecms/area/ui/apos/components/AposWidgetControls.vue +++ b/packages/apostrophe/modules/@apostrophecms/area/ui/apos/components/AposWidgetControls.vue @@ -90,7 +90,6 @@ export default { icon: 'plus-icon', type: 'group', modifiers: [ 'small', 'inline' ], - role: 'menuitem', class: 'apos-area-modify-controls__button', iconSize: 16, disableFocus: !this.tabbable diff --git a/packages/apostrophe/modules/@apostrophecms/doc-type/ui/apos/logic/AposDocContextMenu.js b/packages/apostrophe/modules/@apostrophecms/doc-type/ui/apos/logic/AposDocContextMenu.js index 12e5ddfe86..e766ba75cb 100644 --- a/packages/apostrophe/modules/@apostrophecms/doc-type/ui/apos/logic/AposDocContextMenu.js +++ b/packages/apostrophe/modules/@apostrophecms/doc-type/ui/apos/logic/AposDocContextMenu.js @@ -273,13 +273,13 @@ export default { }); }, moduleName() { - if (apos.modules[this.context.type].action === apos.modules['@apostrophecms/page'].action) { + if (apos.modules[this.context.type]?.action === apos.modules['@apostrophecms/page'].action) { return '@apostrophecms/page'; } return this.context.type; }, moduleOptions() { - return apos.modules[this.moduleName]; + return apos.modules[this.moduleName] || {}; }, isUpdateOperation() { return !!this.context._id; @@ -482,7 +482,9 @@ export default { ); }, preview(doc) { - window.open(doc._url, '_blank').focus(); + // window.open() returns null when a popup blocker intercepts it, + // so guard before calling focus(). + window.open(doc._url, '_blank')?.focus(); }, async copy(doc) { // If there are changes warn the user before discarding them before diff --git a/packages/apostrophe/modules/@apostrophecms/i18n/i18n/en.json b/packages/apostrophe/modules/@apostrophecms/i18n/i18n/en.json index 0adcf393d2..e93b9a68f8 100644 --- a/packages/apostrophe/modules/@apostrophecms/i18n/i18n/en.json +++ b/packages/apostrophe/modules/@apostrophecms/i18n/i18n/en.json @@ -205,6 +205,7 @@ "editImageRelationshipTitle": "Adjust Image", "editRelationship": "Edit Relationship", "editRelationshipFor": "Edit Relationship for {{ title }}", + "editSubform": "Edit {{ label }}", "editType": "Edit {{ type }}", "editWidget": "Edit Widget", "editWidgetForeignTooltip": "Click to edit this content in its natural context", @@ -569,6 +570,7 @@ "richTextHighlight": "Mark", "richTextHorizontalRule": "Horizontal Rule", "richTextHorizontalRuleDescription": "Add a horizontal separator", + "richTextEditor": "Rich text editor", "richTextInsertMenuHeading": "Insert element...", "richTextItalic": "Italic", "richTextLink": "Link", @@ -632,6 +634,7 @@ "slugInUse": "Slug already in use", "someoneElseTookControl": "{{ who }} took control of this document in another tab or window. A document can only be edited in one place at a time.", "splitCell": "Split Cell", + "status": "Status", "style": "Style", "styleAlignment": "Alignment", "styleBackground": "Background", diff --git a/packages/apostrophe/modules/@apostrophecms/image/ui/apos/components/AposMediaManagerEditor.vue b/packages/apostrophe/modules/@apostrophecms/image/ui/apos/components/AposMediaManagerEditor.vue index b6598bcaa1..46eba052d1 100644 --- a/packages/apostrophe/modules/@apostrophecms/image/ui/apos/components/AposMediaManagerEditor.vue +++ b/packages/apostrophe/modules/@apostrophecms/image/ui/apos/components/AposMediaManagerEditor.vue @@ -14,7 +14,7 @@ v-if="activeMedia.attachment && activeMedia.attachment._urls" class="apos-media-editor__thumb" :src="activeMedia.attachment._urls[restoreOnly ? 'one-sixth' : 'one-third']" - :alt="activeMedia.description" + :alt="activeMedia.description || ''" >
    @@ -484,7 +484,7 @@ export default { & { line-height: var(--a-line-tallest); - color: var(--a-base-4); + color: var(--a-base-2); } } diff --git a/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposGridColumn.vue b/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposGridColumn.vue index 87ff04278a..8b13324d76 100644 --- a/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposGridColumn.vue +++ b/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposGridColumn.vue @@ -6,7 +6,6 @@ :style="itemStyles" :class="widgetStyles.classes" class="apos-layout__item" - role="gridcell" data-apos-test="aposLayoutItem" :data-id="item._id" :data-tablet-full="props.tabletFullItems[item._id] || false" diff --git a/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposGridManager.vue b/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposGridManager.vue index e776b9aab8..f7dc396f71 100644 --- a/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposGridManager.vue +++ b/packages/apostrophe/modules/@apostrophecms/layout-widget/ui/apos/components/AposGridManager.vue @@ -14,7 +14,6 @@ :key="item._id" ref="items" class="apos-layout__item" - role="gridcell" data-apos-test="aposManageLayoutItem" :data-id="item._id" :style="{ diff --git a/packages/apostrophe/modules/@apostrophecms/login/ui/apos/components/TheAposLogin.vue b/packages/apostrophe/modules/@apostrophecms/login/ui/apos/components/TheAposLogin.vue index 1bb2223ff3..a22a2a86bf 100644 --- a/packages/apostrophe/modules/@apostrophecms/login/ui/apos/components/TheAposLogin.vue +++ b/packages/apostrophe/modules/@apostrophecms/login/ui/apos/components/TheAposLogin.vue @@ -10,6 +10,7 @@
    {{ $t('apostrophe:loginHome') }}
    -
    +
    +

    + {{ $t('apostrophe:login') }} +