diff --git a/.changeset/array-draggable-false-non-inline.md b/.changeset/array-draggable-false-non-inline.md new file mode 100644 index 0000000000..8d3056d696 --- /dev/null +++ b/.changeset/array-draggable-false-non-inline.md @@ -0,0 +1,5 @@ +--- +"apostrophe": minor +--- + +Added support for `draggable: false` on non-inline `array` schema fields. Previously this option was only respected when `inline: true`. When set on a standard (modal-based) array field, drag-and-drop reordering and keyboard reordering are now disabled in the array editor's slat list. diff --git a/.changeset/common-beans-lie.md b/.changeset/common-beans-lie.md new file mode 100644 index 0000000000..42fd0945d7 --- /dev/null +++ b/.changeset/common-beans-lie.md @@ -0,0 +1,5 @@ +--- +"@apostrophecms/seo": minor +--- + +Removes unimplemented hreflang output; use @apostrophecms/sitemap for hreflang support diff --git a/.changeset/curvy-bobcats-peel.md b/.changeset/curvy-bobcats-peel.md new file mode 100644 index 0000000000..0849faf743 --- /dev/null +++ b/.changeset/curvy-bobcats-peel.md @@ -0,0 +1,5 @@ +--- +"apostrophe": minor +--- + +Introduced support for postgres://, sqlite://, and multipostgres:// database URIs in addition to mongodb://. The new db-connect API supports all of the database operations currently used in our own core, pro and multisite modules. For more information see the documentation. diff --git a/.changeset/env-secrets-support.md b/.changeset/env-secrets-support.md new file mode 100644 index 0000000000..5c79300289 --- /dev/null +++ b/.changeset/env-secrets-support.md @@ -0,0 +1,5 @@ +--- +"apostrophe": minor +--- + +The session secret and the uploadfs `disabledFileKey` can now be supplied via the `APOS_SESSION_SECRET` and `APOS_UPLOADFS_DISABLED_FILE_KEY` environment variables. As with other Apostrophe environment variables, these take precedence over the corresponding `app.js` configuration. diff --git a/.changeset/new-doors-turn.md b/.changeset/new-doors-turn.md new file mode 100644 index 0000000000..b7aa9e76ab --- /dev/null +++ b/.changeset/new-doors-turn.md @@ -0,0 +1,5 @@ +--- +"sanitize-html": patch +--- + +Address a potential vulnerability when nonTextTags is configured in a nonstandard way. While it is never a good idea to remove known non-text tags from the standard list e.g. script, styles, etc., this change ensures that doing so does not result in nested tags being passed through without sanitization when they are not expressly allowed. (ApostropheCMS would never trigger this situation.) Thanks to [Dipanshu singh](https://github.com/Dipanshusinghh) for pointing out the issue and contributing the fix. diff --git a/.changeset/ripe-terms-happen.md b/.changeset/ripe-terms-happen.md new file mode 100644 index 0000000000..4ad5dec883 --- /dev/null +++ b/.changeset/ripe-terms-happen.md @@ -0,0 +1,11 @@ +--- +"@apostrophecms/apostrophe-astro": minor +"apostrophe": minor +--- + +Fixed adding or removing an area field from a schema breaking existing documents on an external front such as Astro. + +- `AposArea` now renders only schema-backed areas. A missing area no longer throws, and an area orphaned by removing its field from the schema (while its content remains in the document) renders nothing instead of breaking sibling areas in edit mode. Logged-in editors get a diagnostic message in place of an orphaned area; anonymous visitors see nothing. +- Editable documents sent to an external front now materialize empty area objects for schema area fields added after the document was created, so they can be edited in context. +- `apos.util.getManagerOf` accepts a `{ log }` option to suppress its error log when probing objects that may not have a manager. + diff --git a/.changeset/shaky-regions-spend.md b/.changeset/shaky-regions-spend.md new file mode 100644 index 0000000000..a21e913af0 --- /dev/null +++ b/.changeset/shaky-regions-spend.md @@ -0,0 +1,5 @@ +--- +"@apostrophecms/seo": minor +--- + +Removes the `seoSiteCanonicalUrl` field from global settings. The base URL is now derived automatically from `APOS_BASE_URL` or the `baseUrl` option. The value remains available at `req.data.global.seoSiteCanonicalUrl` for backwards compatibility. diff --git a/.changeset/smart-kids-rest.md b/.changeset/smart-kids-rest.md new file mode 100644 index 0000000000..ce014a1952 --- /dev/null +++ b/.changeset/smart-kids-rest.md @@ -0,0 +1,5 @@ +--- +"apostrophe": patch +--- + +Selecting an item in a relationship "browse" dialog no longer scrolls the title and Cancel/Select buttons out of view when the item is far down the list. diff --git a/.changeset/soft-hats-smile.md b/.changeset/soft-hats-smile.md new file mode 100644 index 0000000000..3a7897cf1d --- /dev/null +++ b/.changeset/soft-hats-smile.md @@ -0,0 +1,6 @@ +--- +"@apostrophecms/cli": minor +--- + +**Breaking:** `apos create` is now an interactive guided installer (it delegates to `create-apostrophe`). The `` positional argument and the `--starter` and `--mongodb-uri` options have been removed - project name, starter kit, and database are now chosen through prompts. For scripted installs, use `npm create apostrophe@latest -- --unattended` instead. + diff --git a/.changeset/sparkly-experts-walk.md b/.changeset/sparkly-experts-walk.md new file mode 100644 index 0000000000..b641817bfa --- /dev/null +++ b/.changeset/sparkly-experts-walk.md @@ -0,0 +1,5 @@ +--- +"apostrophe": patch +--- + +Sites with a custom filterByIndexPage method no longer experience failures in the sitemap module and potential creeping CPU performance penalties. A regression introduced with our static site support, but not specific to static sites. diff --git a/.changeset/twelve-paws-wink.md b/.changeset/twelve-paws-wink.md new file mode 100644 index 0000000000..3c523198ea --- /dev/null +++ b/.changeset/twelve-paws-wink.md @@ -0,0 +1,5 @@ +--- +"@apostrophecms/redirect": minor +--- + +Prevent infinite redirects to external URLs diff --git a/.changeset/violet-windows-draw.md b/.changeset/violet-windows-draw.md new file mode 100644 index 0000000000..755e3cde7d --- /dev/null +++ b/.changeset/violet-windows-draw.md @@ -0,0 +1,5 @@ +--- +"apostrophe": minor +--- + +JSX support for templates within ApostropheCMS. JSX is now co-equal with Nunjucks, with a gradual migration strategy. Anyone who is familiar with React will be very comfortable writing JSX templates, which also offer a superior debugging experience, and templates can be migrated gradually. JSX is a great option for those who don't wish to create parallel Astro and ApostropheCMS projects, but still prefer a modern syntax. For more information, see the new [JSX templates guide](https://apostrophecms.com/docs/guide/jsx-templates.html). diff --git a/.github/workflows/monorepo.yml b/.github/workflows/monorepo.yml index 84746c945f..8cf21c4887 100644 --- a/.github/workflows/monorepo.yml +++ b/.github/workflows/monorepo.yml @@ -4,9 +4,10 @@ permissions: env: # Define supported runtime versions for matrix expansion once for the workflow. - NODE_VERSIONS_JSON: "[20,22,24]" + NODE_VERSIONS_JSON: "[22,24,26]" MONGODB_VERSIONS_JSON: '["7","8"]' REDIS_VERSION: "7" + POSTGRES_VERSION: "16" on: push: @@ -124,7 +125,7 @@ jobs: run: | month='${{ steps.cache-month.outputs.value }}' mongo_suffix=$(jq -r '.[]' <<< '${{ env.MONGODB_VERSIONS_JSON }}' | paste -sd'-' -) - echo "key=docker-images-${month}-mongo${mongo_suffix}-redis${{ env.REDIS_VERSION }}" >> "$GITHUB_OUTPUT" + echo "key=docker-images-${month}-mongo${mongo_suffix}-redis${{ env.REDIS_VERSION }}-pg${{ env.POSTGRES_VERSION }}" >> "$GITHUB_OUTPUT" - name: Restore docker image cache id: docker-cache @@ -143,6 +144,8 @@ jobs: done docker pull "redis:${{ env.REDIS_VERSION }}" docker save "redis:${{ env.REDIS_VERSION }}" -o ".github/docker-cache/redis-${{ env.REDIS_VERSION }}.tar" + docker pull "postgres:${{ env.POSTGRES_VERSION }}" + docker save "postgres:${{ env.POSTGRES_VERSION }}" -o ".github/docker-cache/postgres-${{ env.POSTGRES_VERSION }}.tar" warm-sharp-cache: name: Warm sharp/libvips cache (Node ${{ matrix.nodeVersion }}) @@ -216,7 +219,7 @@ jobs: du -sh ~/.npm/_libvips || true package-tests: - name: ${{ format('{0} ({1}, {2})', matrix.package, matrix.nodeVersion, matrix.needsMongo && matrix.mongodbVersion || 'n/a') }} + name: ${{ format('{0} ({1}, {2}{3})', matrix.group, matrix.nodeVersion, matrix.adapter || 'n/a', matrix.needsMongo && format(' mongo {0}', matrix.mongodbVersion) || '') }} needs: - setup - shared-runtime @@ -279,7 +282,7 @@ jobs: - name: Restore docker image cache id: docker-cache-restore uses: actions/cache/restore@v4 - if: matrix.needsMongo || matrix.needsRedis + if: matrix.needsMongo || matrix.needsRedis || matrix.needsPostgres with: path: .github/docker-cache key: ${{ needs.shared-runtime.outputs.docker-cache-key }} @@ -300,6 +303,14 @@ jobs: if: steps.docker-cache-restore.outputs.cache-hit != 'true' && matrix.needsRedis run: docker pull redis:${{ env.REDIS_VERSION }} + - name: Load cached Postgres image + if: steps.docker-cache-restore.outputs.cache-hit == 'true' && matrix.needsPostgres + run: docker load -i ".github/docker-cache/postgres-${{ env.POSTGRES_VERSION }}.tar" + + - name: Pull Postgres image (cache miss) + if: steps.docker-cache-restore.outputs.cache-hit != 'true' && matrix.needsPostgres + run: docker pull postgres:${{ env.POSTGRES_VERSION }} + - name: Start MongoDB if: matrix.needsMongo run: | @@ -358,15 +369,64 @@ jobs: docker logs redis exit 1 + - name: Start PostgreSQL + if: matrix.needsPostgres + run: | + docker rm -f postgres >/dev/null 2>&1 || true + docker run -d \ + --name postgres \ + --publish 5432:5432 \ + -e POSTGRES_HOST_AUTH_METHOD=trust \ + --health-cmd "pg_isready -U postgres" \ + --health-interval 5s \ + --health-timeout 5s \ + --health-retries 12 \ + postgres:${{ env.POSTGRES_VERSION }} + echo "Waiting for PostgreSQL to report healthy..." + for attempt in $(seq 1 60); do + status=$(docker inspect --format='{{.State.Health.Status}}' postgres 2>/dev/null || echo "starting") + if [ "$status" = "healthy" ]; then + exit 0 + fi + if [ "$status" = "unhealthy" ]; then + echo "PostgreSQL reported unhealthy" >&2 + docker logs postgres + exit 1 + fi + sleep 2 + done + echo "PostgreSQL failed to become healthy in time" >&2 + docker logs postgres + exit 1 + - name: Install workspace dependencies run: pnpm install --frozen-lockfile - name: Run package tests - run: pnpm run --filter "${{ matrix.package }}" --if-present test + run: | + failed=0 + for pkg in $(echo '${{ matrix.packages }}' | jq -r '.[]'); do + echo "::group::Testing $pkg" + if ! pnpm run --filter "$pkg" --if-present test; then + echo "::error::Tests failed for $pkg" + failed=1 + fi + echo "::endgroup::" + done + exit $failed env: CI: true # Yes we want import-export to test with automatic-translation TEST_WITH_PRO: "1" + # Adapter selection: mongodb (default), postgres, or sqlite. + # ADAPTER is used by db-connect tests, APOS_TEST_DB_PROTOCOL by apostrophe tests. + ADAPTER: ${{ matrix.adapter }} + APOS_TEST_DB_PROTOCOL: ${{ matrix.adapter }} + PGUSER: postgres + + - name: Stop PostgreSQL + if: always() && matrix.needsPostgres + run: docker rm -f postgres || true - name: Stop Redis if: always() && matrix.needsRedis diff --git a/.github/workflows/scripts/detect-impacted-packages.mjs b/.github/workflows/scripts/detect-impacted-packages.mjs index 453f370938..392a29b6d1 100644 --- a/.github/workflows/scripts/detect-impacted-packages.mjs +++ b/.github/workflows/scripts/detect-impacted-packages.mjs @@ -73,7 +73,8 @@ async function main() { package: name, directory: packages.get(name).relativeDir, requiresMongo: packages.get(name).requiresMongo !== false, - requiresRedis: packages.get(name).requiresRedis === true + requiresRedis: packages.get(name).requiresRedis === true, + mongodbOnly: packages.get(name).mongodbOnly === true })) }; @@ -130,6 +131,7 @@ async function loadPackages() { const testConfig = manifest.apostropheTestConfig || {}; const requiresMongo = testConfig.requiresMongo !== false; const requiresRedis = testConfig.requiresRedis === true; + const mongodbOnly = testConfig.mongodbOnly === true; map.set(manifest.name, { name: manifest.name, @@ -137,7 +139,8 @@ async function loadPackages() { dependencies, hasTestScript, requiresMongo, - requiresRedis + requiresRedis, + mongodbOnly }); })); diff --git a/.github/workflows/scripts/expand-runtime-matrix.mjs b/.github/workflows/scripts/expand-runtime-matrix.mjs index 0c0e46f690..3672fad06b 100644 --- a/.github/workflows/scripts/expand-runtime-matrix.mjs +++ b/.github/workflows/scripts/expand-runtime-matrix.mjs @@ -1,6 +1,21 @@ #!/usr/bin/env node -// Expands the impacted package matrix with runtime permutations supplied +// Expands the impacted package matrix with runtime permutations supplied // via env vars. +// +// Packages are grouped into jobs to stay within GitHub's 256-entry matrix limit: +// - "apostrophe" runs solo (the main package, benefits from its own status). +// - All other database packages are grouped into an "ecosystem" job. +// - mongodbOnly packages are grouped into "ecosystem-mongodb". +// - Non-database packages are grouped into "standalone". +// +// For groups that need a database, three adapter variants are emitted: +// 1. mongodb – all Node versions × all MongoDB versions +// 2. postgres – latest LTS Node only, no MongoDB +// 3. sqlite – latest LTS Node only, no MongoDB +// +// The latest LTS Node version is the highest even-numbered entry in +// NODE_VERSIONS_JSON. + import { readFile } from 'fs/promises'; const args = process.argv.slice(2); @@ -36,25 +51,120 @@ function parseJsonArray(name, raw) { const nodeVersions = parseJsonArray('NODE_VERSIONS_JSON', process.env.NODE_VERSIONS_JSON); const mongodbVersions = parseJsonArray('MONGODB_VERSIONS_JSON', process.env.MONGODB_VERSIONS_JSON); +// Latest LTS = highest even-numbered Node version +const latestLts = [...nodeVersions] + .filter((v) => Number(v) % 2 === 0) + .sort((a, b) => Number(b) - Number(a))[0]; + const impact = JSON.parse(await readFile(impactPath, 'utf8')); const packages = impact?.matrix?.include || []; -const include = []; + +// The main apostrophe package always gets its own jobs for clear CI status. +const SOLO_PACKAGES = new Set(['apostrophe']); + +// Sort packages into groups +const solo = []; +const ecosystem = []; +const ecosystemMongodbOnly = []; +const standalone = []; for (const pkg of packages) { - const needsMongo = pkg.requiresMongo !== false; - const needsRedis = pkg.requiresRedis === true; - const mongoTargets = needsMongo ? mongodbVersions : ['']; + const needsDb = pkg.requiresMongo !== false; + if (SOLO_PACKAGES.has(pkg.package)) { + solo.push(pkg); + } else if (needsDb && pkg.mongodbOnly) { + ecosystemMongodbOnly.push(pkg); + } else if (needsDb) { + ecosystem.push(pkg); + } else { + standalone.push(pkg); + } +} + +const include = []; + +// Emit runtime combinations for a group of packages. +function emitGroup(group, pkgs) { + if (!pkgs.length) { + return; + } + const packageNames = JSON.stringify(pkgs.map((p) => p.package)); + const needsRedis = pkgs.some((p) => p.requiresRedis === true); + const mongodbOnly = pkgs.every((p) => p.mongodbOnly); + + // mongodb: all Node versions × all MongoDB versions for (const nodeVersion of nodeVersions) { - for (const mongodbVersion of mongoTargets) { + for (const mongodbVersion of mongodbVersions) { include.push({ - ...pkg, + group, + packages: packageNames, nodeVersion, mongodbVersion, - needsMongo, + adapter: 'mongodb', + needsMongo: true, + needsPostgres: false, needsRedis }); } } + // postgres and sqlite: latest LTS only, skip for mongodb-only groups + if (!mongodbOnly) { + include.push({ + group, + packages: packageNames, + nodeVersion: latestLts, + mongodbVersion: '', + adapter: 'postgres', + needsMongo: false, + needsPostgres: true, + needsRedis + }); + include.push({ + group, + packages: packageNames, + nodeVersion: latestLts, + mongodbVersion: '', + adapter: 'sqlite', + needsMongo: false, + needsPostgres: false, + needsRedis + }); + } +} + +// Emit non-database group (no adapter variants, just Node versions) +function emitStandalone(group, pkgs) { + if (!pkgs.length) { + return; + } + const packageNames = JSON.stringify(pkgs.map((p) => p.package)); + for (const nodeVersion of nodeVersions) { + include.push({ + group, + packages: packageNames, + nodeVersion, + mongodbVersion: '', + adapter: '', + needsMongo: false, + needsPostgres: false, + needsRedis: false + }); + } +} + +// Solo packages each get their own group +for (const pkg of solo) { + emitGroup(pkg.package, [pkg]); +} + +if (ecosystem.length) { + emitGroup('ecosystem', ecosystem); +} +if (ecosystemMongodbOnly.length) { + emitGroup('ecosystem-mongodb', ecosystemMongodbOnly); +} +if (standalone.length) { + emitStandalone('standalone', standalone); } process.stdout.write(JSON.stringify({ include })); diff --git a/.gitignore b/.gitignore index 6bccc37b4b..186dae1605 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ public/apos-frontend .DS_Store coverage/ .nyc_output +claude-tools/logs/ +.claude +specs/ diff --git a/claude-tools/check-demo-jsx.mjs b/claude-tools/check-demo-jsx.mjs new file mode 100644 index 0000000000..518fc60019 --- /dev/null +++ b/claude-tools/check-demo-jsx.mjs @@ -0,0 +1,49 @@ +// Quick syntax check for every .jsx template under public-demo. Loads +// each via the same JSX loader used in production and reports compile +// errors with proper file/line info. +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; +import fs from 'node:fs'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const apostropheRoot = path.resolve(here, '..'); +const require = createRequire(path.join(apostropheRoot, 'packages/apostrophe/index.js')); + +const { install } = require(path.join(apostropheRoot, 'packages/apostrophe/modules/@apostrophecms/template/lib/jsxLoader.js')); +install(); + +const demoRoot = '/srv/workspace/apostrophecms/public-demo'; + +function* walk(dir) { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (entry.name === 'node_modules' || entry.name === 'data') continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + yield* walk(full); + } else if (entry.name.endsWith('.jsx')) { + yield full; + } + } +} + +let failed = 0; +for (const file of walk(demoRoot)) { + try { + require(file); + console.log('OK ', file); + } catch (err) { + failed += 1; + console.error('FAIL', file); + console.error(' ', err.message); + if (err.stack) { + console.error(err.stack.split('\n').slice(1, 5).join('\n')); + } + } +} + +if (failed > 0) { + console.error(`\n${failed} file(s) failed to compile/load.`); + process.exit(1); +} +console.log(`\nAll JSX templates loaded successfully.`); diff --git a/claude-tools/run-core-tests.sh b/claude-tools/run-core-tests.sh new file mode 100755 index 0000000000..eb8b77c009 --- /dev/null +++ b/claude-tools/run-core-tests.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Run the apostrophe core test suite against a chosen DB adapter and log +# output to claude-tools/logs/core-.log. Usage: +# +# ./claude-tools/run-core-tests.sh mongodb +# ./claude-tools/run-core-tests.sh postgres +# ./claude-tools/run-core-tests.sh sqlite +# +# NEVER run multiple adapters in parallel — the test suite is not designed +# for concurrent runs and the host has limited resources. + +set -u +adapter="${1:-}" +if [[ -z "$adapter" ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +root="$(cd "$(dirname "$0")/.." && pwd)" +logdir="$root/claude-tools/logs" +mkdir -p "$logdir" +log="$logdir/core-$adapter.log" +: > "$log" + +echo "=== $adapter core tests ($(date -Is)) ===" | tee -a "$log" + +cd "$root/packages/apostrophe" + +extra=() +if [[ "$adapter" == "postgres" ]]; then + extra=(env PGPASSWORD=testpassword) +fi + +APOS_TEST_DB_PROTOCOL="$adapter" "${extra[@]}" npm run test:base >> "$log" 2>&1 +code=$? +echo "=== exit=$code ===" | tee -a "$log" +exit "$code" diff --git a/package.json b/package.json index 1fd8ccdf94..c80aee125e 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "dev": "pnpm --parallel --recursive run dev", "build": "pnpm --recursive run build", "lint": "pnpm --recursive run lint", - "test": "pnpm --recursive run test", + "test": "APOS_TEST_DB_PROTOCOL=postgres pnpm run test:main && APOS_TEST_DB_PROTOCOL=mongodb pnpm run test:main && APOS_TEST_DB_PROTOCOL=sqlite pnpm run test:main && APOS_TEST_DB_PROTOCOL=multipostgres pnpm run test:main", + "test:main": "echo \"APOS_TEST_DB_PROTOCOL IS: $APOS_TEST_DB_PROTOCOL\" && pnpm --recursive run test", "eslint": "pnpm --recursive run eslint", "mocha": "pnpm --recursive run mocha", "clean": "pnpm -r exec rm -rf node_modules && rm -rf node_modules && rm pnpm-lock.yaml" @@ -18,6 +19,7 @@ }, "pnpm": { "onlyBuiltDependencies": [ + "better-sqlite3", "sharp", "vue-demi", "@parcel/watcher", diff --git a/packages/apostrophe-astro/components/AposArea.astro b/packages/apostrophe-astro/components/AposArea.astro index 418b01d45d..a790b34bbb 100644 --- a/packages/apostrophe-astro/components/AposArea.astro +++ b/packages/apostrophe-astro/components/AposArea.astro @@ -13,9 +13,22 @@ const { let attributes = {}; -const widgets = area?.items || []; +// Enough structure to render as an area at all (hardens against malformed +// data — never crash the render). +const renderable = area?.metaType === "area" && Array.isArray(area?.items); -const isEdit = area?._edit && Astro.url.searchParams.get("aposEdit"); +const isOrphan = Boolean(area?._isOrphan); +const isArea = renderable && !isOrphan; + +const hasField = Boolean(area?.field); + +// Defensive: a corrupt item (null/typeless) must never crash the render. +const widgets: Record[] = (area?.items || []).filter( + (item: any) => item && item.type, +); + +const isEdit = + hasField && area?._edit && Astro.url.searchParams.get("aposEdit"); const forceWrapper = aposAttributes || aposStyle || aposClassName; const WidgetComponent = widgetComponent ?? AposWidget; @@ -43,16 +56,16 @@ if (isEdit) { }; } const Wrapper = isEdit || forceWrapper ? "div" : Fragment; -const widgetOptions = getWidgetOptions(area.options); +const widgetOptions = getWidgetOptions(area?.options); -function getWidgetOptions(options) { - let widgets = options.widgets || {}; +function getWidgetOptions(options: any = {}) { + let widgets = { ...(options.widgets || {}) }; if (options.groups) { for (const group of Object.keys(options.groups)) { widgets = { ...widgets, - ...options.groups[group].widgets, + ...(options.groups[group]?.widgets || {}), }; } } @@ -60,22 +73,47 @@ function getWidgetOptions(options) { } --- - - { - widgets?.map((item) => { - const options = { - ...item._options, - ...widgetOptions[item.type], - }; - return ( - - ); - }) - } - +{ + isArea ? ( + + {widgets.map((item) => { + const options = { + ...item._options, + ...widgetOptions[item.type], + }; + return ( + + ); + })} + + ) : ( + // Genuine orphan only. Dev-only diagnostic: `import.meta.env.DEV` is + // replaced with `false` in production, so this branch is dead-code + // eliminated. Un-annotated (e.g. REST-delivered) areas never reach here — + // they render above. + (import.meta as any).env.DEV && + isOrphan && ( +
+ ApostropheCMS: an area passed to{" "} + {""} is not defined in the schema, so it was + not rendered. Its field was likely removed from the schema while content + remains in the document. Restore the field in the schema, or remove the + matching {""} from this template. +
+ + area _id: {area._id} — document: {area._docId} + +
+ ) + ) +} diff --git a/packages/apostrophe/.gitignore b/packages/apostrophe/.gitignore index 894a9d6ba3..16ad414a99 100644 --- a/packages/apostrophe/.gitignore +++ b/packages/apostrophe/.gitignore @@ -42,3 +42,6 @@ test/public/uploads # vim swp files .*.sw* + +# claude-tools log files +claude-tools/**/*.log diff --git a/packages/apostrophe/claude-tools/detect-handles.js b/packages/apostrophe/claude-tools/detect-handles.js new file mode 100644 index 0000000000..9cf711d976 --- /dev/null +++ b/packages/apostrophe/claude-tools/detect-handles.js @@ -0,0 +1,46 @@ +// Require this before running mocha to detect what activates process.stdin +// Usage: npx mocha -t 10000 --require ./claude-tools/detect-handles.js test/assets.js + +console.log(`stdin paused at startup: ${process.stdin.isPaused()}`); +console.log(`stdin readableFlowing at startup: ${process.stdin.readableFlowing}`); + +// Monkey-patch stdin.resume to capture the call stack +const origResume = process.stdin.resume.bind(process.stdin); +process.stdin.resume = function(...args) { + console.log('\n=== process.stdin.resume() called ==='); + console.log(new Error().stack); + return origResume(...args); +}; + +// Monkey-patch stdin.on to detect 'data' listener additions +const origOn = process.stdin.on.bind(process.stdin); +process.stdin.on = function(event, ...args) { + if (event === 'data' || event === 'readable') { + console.log(`\n=== process.stdin.on('${event}') called ===`); + console.log(new Error().stack); + } + return origOn(event, ...args); +}; + +// Periodically check stdin state changes +let lastState = process.stdin.readableFlowing; +const checker = setInterval(() => { + if (process.stdin.readableFlowing !== lastState) { + console.log(`\n=== stdin readableFlowing changed: ${lastState} -> ${process.stdin.readableFlowing} ===`); + console.log(new Error().stack); + lastState = process.stdin.readableFlowing; + } +}, 100); +checker.unref(); + +const origRun = require('mocha/lib/runner').prototype.run; +require('mocha/lib/runner').prototype.run = function(fn) { + return origRun.call(this, function(failures) { + console.log(`\nstdin paused at end: ${process.stdin.isPaused()}`); + console.log(`stdin readableFlowing at end: ${process.stdin.readableFlowing}`); + setTimeout(() => { + process.exit(failures ? 3 : 0); + }, 2000); + if (fn) fn(failures); + }); +}; diff --git a/packages/apostrophe/claude-tools/minimal-hang-test.js b/packages/apostrophe/claude-tools/minimal-hang-test.js new file mode 100644 index 0000000000..3d29952b7f --- /dev/null +++ b/packages/apostrophe/claude-tools/minimal-hang-test.js @@ -0,0 +1,28 @@ +// Minimal test to isolate what causes the hang. +// Must reference the test/ directory as root for proper module resolution. +const t = require('../test-lib/test.js'); +const path = require('path'); + +// Fake a module object rooted in test/ like the real tests do +const fakeModule = { + id: path.join(__dirname, '../test/fake'), + filename: path.join(__dirname, '../test/fake.js'), + paths: [path.join(__dirname, '../test/node_modules')] +}; + +describe('Minimal hang test', function() { + this.timeout(60000); + let apos; + + after(async function() { + await t.destroy(apos); + console.log('after: destroy complete'); + }); + + it('should create and use apos without hanging', async function() { + apos = await t.create({ + root: fakeModule + }); + console.log('apos created successfully'); + }); +}); diff --git a/packages/apostrophe/claude-tools/mongo-close-test.js b/packages/apostrophe/claude-tools/mongo-close-test.js new file mode 100644 index 0000000000..360ce44299 --- /dev/null +++ b/packages/apostrophe/claude-tools/mongo-close-test.js @@ -0,0 +1,11 @@ +// Test whether a MongoDB connection keeps the process alive after close() +const mongoConnect = require('../../../packages/db-connect/lib/mongodb-connect'); + +(async () => { + const uri = 'mongodb://localhost:27017/test_handle_leak'; + console.log('Connecting...'); + const client = await mongoConnect(uri); + console.log('Connected. Closing...'); + await client.close(); + console.log('Closed. Process should exit now if no leaked handles.'); +})(); diff --git a/packages/apostrophe/claude-tools/stdin-ref-test.js b/packages/apostrophe/claude-tools/stdin-ref-test.js new file mode 100644 index 0000000000..d562211bcb --- /dev/null +++ b/packages/apostrophe/claude-tools/stdin-ref-test.js @@ -0,0 +1,14 @@ +// Check if process.stdin keeps the process alive +// If this script hangs, stdin is ref'd. If it exits, stdin is unref'd. + +console.log(`stdin isTTY: ${process.stdin.isTTY}`); +console.log(`stdin readableFlowing: ${process.stdin.readableFlowing}`); +console.log(`stdin isPaused: ${process.stdin.isPaused()}`); + +// Check ref status +if (typeof process.stdin.unref === 'function') { + console.log('stdin has unref method'); +} + +console.log('Waiting to see if process exits on its own...'); +// Don't do anything - just see if the process exits diff --git a/packages/apostrophe/eslint.config.js b/packages/apostrophe/eslint.config.js index 9377dc1974..5169f67265 100644 --- a/packages/apostrophe/eslint.config.js +++ b/packages/apostrophe/eslint.config.js @@ -7,7 +7,9 @@ module.exports = defineConfig([ '**/blueimp/**/*.js', 'test/public', 'test/apos-build', - 'coverage' + 'test/modules/jsx-mixed-test/views/syntax-error.jsx', + 'coverage', + 'claude-tools' ]), apostrophe ]); diff --git a/packages/apostrophe/modules/@apostrophecms/area/index.js b/packages/apostrophe/modules/@apostrophecms/area/index.js index a96d8248c8..4ec664f70e 100644 --- a/packages/apostrophe/modules/@apostrophecms/area/index.js +++ b/packages/apostrophe/modules/@apostrophecms/area/index.js @@ -140,7 +140,7 @@ module.exports = { // so this logic is reproduced partially self.apos.doc.walk(area, (o, k, v) => { if (v && v.metaType === 'area') { - const manager = self.apos.util.getManagerOf(o); + const manager = self.apos.util.getManagerOf(o, { log: false }); if (!manager) { self.apos.util.warnDevOnce( 'noManagerForDocInExternalFront', @@ -285,6 +285,95 @@ module.exports = { self.missingWidgetTypes[name] = true; } }, + // Build an empty area and attach it to `parent[name]`. When the area's + // location can be resolved inside the *persisted* document, also stub it + // into the database so the backend recognizes it for later edits. + // Returns the area. + // + // Options: + // - `throwIfNotFound` (default `false`): when `parent` is doc-backed but + // the document or the container cannot be located in the database, + // throw a `notfound` error instead of returning an in-memory-only + // stub. The `{% area %}` tag opts in to preserve its historical + // behavior; the external front annotator leaves it off so a render is + // never brought down by such a case. + // + // Used by the `{% area %}` tag and the external front annotator as the + // single source of truth for stubbing schema areas that have no value + // yet. + async addMissingArea(parent, name, { throwIfNotFound = false } = {}) { + const area = { + metaType: 'area', + _id: self.apos.util.generateId(), + items: [] + }; + parent[name] = area; + + const docId = parent._docId ?? + (parent.metaType === 'doc' ? parent._id : null); + const areaDotPath = await self.resolvePersistedAreaDotPath(parent, name); + if (!areaDotPath) { + if (throwIfNotFound && docId) { + throw self.apos.error('notfound'); + } + return area; + } + + const result = await self.apos.doc.db.updateOne( + { + _id: docId, + // Idempotent and race-safe: only write when still absent. + [areaDotPath]: { $eq: null } + }, + { + $set: { [areaDotPath]: self.apos.util.clonePermanent(area) } + } + ); + if (result.modifiedCount === 0) { + // Another request stubbed it first (or it already existed): adopt + // the persisted `_id` so we render the same area. + const refreshed = await self.apos.doc.db.findOne({ _id: docId }); + const persisted = refreshed && self.apos.util.get(refreshed, areaDotPath); + if (persisted?._id) { + area._id = persisted._id; + } + } + return area; + }, + + // Resolve the MongoDB dot-path at which `parent[name]` should be stored, + // computed from the *persisted* document so it always reflects real + // storage (not the in-memory graph with its loaded relationships). The + // `parent` object is located inside the freshly read document by its + // `_id`. Returns the dot-path string, or `null` when the area cannot be + // safely persisted (no doc id, doc not in the database, or `parent` is + // not part of the persisted document, e.g. loaded relationship data). + async resolvePersistedAreaDotPath(parent, name) { + const docId = parent._docId ?? + (parent.metaType === 'doc' ? parent._id : null); + if (!docId) { + return null; + } + const mainDoc = await self.apos.doc.db.findOne({ _id: docId }); + if (!mainDoc) { + return null; + } + if (parent._id === docId) { + return name; + } + if (!parent._id) { + return null; + } + const found = self.apos.util.findNestedObjectAndDotPathById( + mainDoc, + parent._id, + { ignoreDynamicProperties: true } + ); + if (!found) { + return null; + } + return `${found.dotPath}.${name}`; + }, prepForRender(area, context, fieldName) { const manager = self.apos.util.getManagerOf(context); const field = manager.schema.find(field => field.name === fieldName); @@ -451,7 +540,10 @@ module.exports = { // Loop over the docs in the array passed in. for (const doc of within) { if (self.apos.externalFrontKey) { - self.apos.template.annotateDocForExternalFront(doc, { scene: req.scene }); + await self.apos.template.annotateDocForExternalFront( + doc, + { scene: req.scene } + ); } const rendered = []; diff --git a/packages/apostrophe/modules/@apostrophecms/area/lib/custom-tags/area.js b/packages/apostrophe/modules/@apostrophecms/area/lib/custom-tags/area.js index 5b8bb7f5fb..865bd68a7b 100644 --- a/packages/apostrophe/modules/@apostrophecms/area/lib/custom-tags/area.js +++ b/packages/apostrophe/modules/@apostrophecms/area/lib/custom-tags/area.js @@ -59,46 +59,7 @@ module.exports = function(self) { } area = doc[name]; if (!area) { - // Problem: area is in schema but that doesn't guarantee it - // has a value, for instance the field could be new in the schema. - // But we need an area _id. Stub it into the db on the fly - // without race conditions - area = { - metaType: 'area', - _id: self.apos.util.generateId(), - items: [] - }; - doc[name] = area; - const docId = doc._docId || ((doc.metaType === 'doc') ? doc._id : null); - if (docId) { - let mainDoc = await self.apos.doc.db.findOne({ _id: docId }); - if (!mainDoc) { - throw self.apos.error('notfound'); - } - let docDotPath; - try { - docDotPath = (doc._id === docId) ? '' : self.apos.util.findNestedObjectAndDotPathById(mainDoc, doc._id).dotPath; - } catch (e) { - // Race condition: someone removed the area's parent object. - // Unlikely thanks to advisory locking - throw self.apos.error('notfound'); - } - const areaDotPath = docDotPath ? `${docDotPath}.${name}` : name; - await self.apos.doc.db.updateOne({ - _id: docId, - // Prevent race condition - [areaDotPath]: { - $eq: null - } - }, { - $set: { - [areaDotPath]: self.apos.util.clonePermanent(area) - } - }); - mainDoc = await self.apos.doc.db.findOne({ _id: docId }); - // Prevent race condition - area._id = self.apos.util.get(mainDoc, areaDotPath)._id; - } + area = await self.apos.area.addMissingArea(doc, name, { throwIfNotFound: true }); } const manager = self.apos.util.getManagerOf(doc); const field = manager.schema.find(field => field.name === name); diff --git a/packages/apostrophe/modules/@apostrophecms/attachment/index.js b/packages/apostrophe/modules/@apostrophecms/attachment/index.js index da44ba9052..63441912b9 100644 --- a/packages/apostrophe/modules/@apostrophecms/attachment/index.js +++ b/packages/apostrophe/modules/@apostrophecms/attachment/index.js @@ -656,7 +656,10 @@ module.exports = { // to avoid template errors getMissingAttachmentUrl() { const defaultIconUrl = '/modules/@apostrophecms/attachment/img/missing-icon.svg'; - self.apos.util.warn('Template warning: Impossible to retrieve the attachment url since it is missing, a default icon has been set. Please fix this ASAP!'); + const e = new Error(); + self.apos.util.warn('Template warning: Impossible to retrieve the attachment url since it is missing, a default icon has been set. Please fix this ASAP!\n\n' + + e.stack + ); // Convert static asset path to full URL, which matters when static // assets are in uploadfs return self.apos.asset.url(defaultIconUrl); diff --git a/packages/apostrophe/modules/@apostrophecms/db/index.js b/packages/apostrophe/modules/@apostrophecms/db/index.js index 8894adba94..58fac171ff 100644 --- a/packages/apostrophe/modules/@apostrophecms/db/index.js +++ b/packages/apostrophe/modules/@apostrophecms/db/index.js @@ -4,11 +4,12 @@ // // ### `uri` // -// The MongoDB connection URI. See the [MongoDB URI documentation](https://docs.mongodb.com/manual/reference/connection-string/). +// The databse connection URI. See the [MongoDB URI documentation](https://docs.mongodb.com/manual/reference/connection-string/) +// and the postgres documentation. // // ### `connect` // -// If present, this object is passed on as options to MongoDB's "connect" +// If present, this object is passed on as options to the database adapters "connect" // method, along with the uri. See the [MongoDB connect settings documentation](http://mongodb.github.io/node-mongodb-native/2.2/reference/connecting/connection-settings/). // // By default, Apostrophe sets options to retry lost connections forever, @@ -20,9 +21,16 @@ // // ### `client` // -// An existing MongoDB connection (MongoClient) object. If present, it is used +// An existing MongoDB-compatible client object. If present, it is used // and `uri`, `host`, `connect`, etc. are ignored. // +// ### `adapters` +// +// An array of adapters, each of which must provide `name`, `connect(uri, options)`, +// and `protocols` properties. `name` may be used to override a core adapter, +// such as `postgres` or `mongodb`. `connect` must resolve to a client object +// supporting a sufficient subset of the mongodb API. +// // ### `versionCheck` // // If `true`, check to make sure the database does not belong to an @@ -49,15 +57,15 @@ // in your project. However you may find it easier to just use the // `client` option. -const mongodbConnect = require('../../../lib/mongodb-connect'); -const escapeHost = require('../../../lib/escape-host'); +const dbConnect = require('@apostrophecms/db-connect'); +const escapeHost = require('../../../lib/escape-host.js'); module.exports = { options: { versionCheck: true }, async init(self) { - await self.connectToMongo(); + await self.connectToDb(); await self.versionCheck(); }, handlers(self) { @@ -81,14 +89,12 @@ module.exports = { }, methods(self) { return { - // Open the database connection. Always uses MongoClient with its - // sensible defaults. Builds a URI if necessary, so we can call it - // in a consistent way. - // - // One default we override: if the connection is lost, we keep - // attempting to reconnect forever. This is the most sensible behavior - // for a persistent process that requires MongoDB in order to operate. - async connectToMongo() { + // Connect to the database and sets self.apos.dbClient + // and self.apos.db. Builds a mongodb URI by default, + // accepting host, port, user, password and name options + // if present. More typically a URI is specified via + // APOS_DB_URI, or via APOS_MONGODB_URI for bc. + async connectToDb() { if (self.options.client) { // Reuse a single client connection http://mongodb.github.io/node-mongodb-native/2.2/api/Db.html#db self.apos.dbClient = self.options.client; @@ -96,32 +102,67 @@ module.exports = { self.connectionReused = true; return; } - let uri = 'mongodb://'; - if (process.env.APOS_MONGODB_URI) { - uri = process.env.APOS_MONGODB_URI; + let uri; + const viaEnv = process.env.APOS_DB_URI || process.env.APOS_MONGODB_URI; + if (viaEnv) { + uri = viaEnv; } else if (self.options.uri) { uri = self.options.uri; } else { - if (self.options.user) { - uri += self.options.user + ':' + self.options.password + '@'; - } - if (!self.options.host) { - self.options.host = 'localhost'; - } - if (!self.options.port) { - self.options.port = 27017; + const validAdapters = [ 'mongodb', 'sqlite', 'postgres', 'multipostgres' ]; + const adapter = process.env.APOS_DEFAULT_DB_ADAPTER || self.options.defaultAdapter || 'mongodb'; + if (!validAdapters.includes(adapter)) { + throw new Error(`Invalid defaultAdapter: "${adapter}". Must be one of: ${validAdapters.join(', ')}`); } if (!self.options.name) { self.options.name = self.apos.shortName; } - uri += escapeHost(self.options.host) + ':' + self.options.port + '/' + self.options.name; + if (adapter === 'sqlite') { + const path = require('path'); + uri = `sqlite://${path.resolve(self.apos.rootDir, 'data', self.options.name + '.sqlite')}`; + } else { + const credentials = self.options.user + ? encodeURIComponent(self.options.user) + ':' + encodeURIComponent(self.options.password) + '@' + : ''; + if (adapter === 'mongodb') { + if (!self.options.host) { + self.options.host = 'localhost'; + } + if (!self.options.port) { + self.options.port = 27017; + } + uri = 'mongodb://' + credentials + escapeHost(self.options.host) + ':' + self.options.port + '/' + self.options.name; + } else { + // postgres or multipostgres + if (!self.options.host) { + self.options.host = 'localhost'; + } + if (!self.options.port) { + self.options.port = 5432; + } + uri = adapter + '://' + credentials + escapeHost(self.options.host) + ':' + self.options.port + '/' + self.options.name; + } + } } - self.apos.dbClient = await mongodbConnect(uri, self.options.connect); + self.apos.dbClient = await dbConnect(uri, { + ...self.options.connect, + adapters: self.options.adapters + }); self.uri = uri; // Automatically uses the db name in the connection string self.apos.db = self.apos.dbClient.db(); }, + // Connect to a database using the appropriate adapter based on the URI protocol. + // Returns a client object compatible with the MongoDB driver interface. + // This method has no side effects — it does not set apos.db or apos.dbClient. + // It can be used to make temporary connections, e.g. for dropping a test database. + async connectToAdapter(uri, options) { + return dbConnect(uri, { + ...options, + adapters: self.options.adapters + }); + }, async versionCheck() { if (!self.options.versionCheck) { return; diff --git a/packages/apostrophe/modules/@apostrophecms/express/index.js b/packages/apostrophe/modules/@apostrophecms/express/index.js index 85165f6b04..00cc95f503 100644 --- a/packages/apostrophe/modules/@apostrophecms/express/index.js +++ b/packages/apostrophe/modules/@apostrophecms/express/index.js @@ -579,6 +579,8 @@ module.exports = { name: self.apos.shortName + '.sid', cookie: {} }); + // Env overrides config, per Apostrophe convention. + sessionOptions.secret = process.env.APOS_SESSION_SECRET || sessionOptions.secret; _.defaults(sessionOptions.cookie, { path: '/', httpOnly: true, diff --git a/packages/apostrophe/modules/@apostrophecms/http/index.js b/packages/apostrophe/modules/@apostrophecms/http/index.js index f7a7b04797..a715259ca3 100644 --- a/packages/apostrophe/modules/@apostrophecms/http/index.js +++ b/packages/apostrophe/modules/@apostrophecms/http/index.js @@ -2,7 +2,7 @@ const _ = require('lodash'); const qs = require('qs'); const fetch = require('node-fetch'); const tough = require('tough-cookie'); -const escapeHost = require('../../../lib/escape-host'); +const escapeHost = require('../../../lib/escape-host.js'); const util = require('util'); module.exports = { diff --git a/packages/apostrophe/modules/@apostrophecms/image/ui/apos/components/AposMediaUploader.vue b/packages/apostrophe/modules/@apostrophecms/image/ui/apos/components/AposMediaUploader.vue index 9ba38df9e4..4041ccd6ea 100644 --- a/packages/apostrophe/modules/@apostrophecms/image/ui/apos/components/AposMediaUploader.vue +++ b/packages/apostrophe/modules/@apostrophecms/image/ui/apos/components/AposMediaUploader.vue @@ -252,6 +252,9 @@ export default { @include apos-transition(); & { + // Contain the visually hidden (`.apos-sr-only`, position: absolute) + // file input so focusing it does not scroll a distant ancestor. + position: relative; display: flex; box-sizing: border-box; align-items: center; diff --git a/packages/apostrophe/modules/@apostrophecms/job/index.js b/packages/apostrophe/modules/@apostrophecms/job/index.js index eeda4a2f14..425e5a85d4 100644 --- a/packages/apostrophe/modules/@apostrophecms/job/index.js +++ b/packages/apostrophe/modules/@apostrophecms/job/index.js @@ -244,7 +244,9 @@ module.exports = { }, setTotal (n) { total = n; - return self.setTotal(job, n); + const result = self.setTotal(job, n); + promises.push(result); + return result; }, setResults (_results) { results = _results; @@ -412,12 +414,12 @@ module.exports = { // // No promise is returned as this method just updates // the job tracking information in the background. - setTotal(job, total) { - self.db.updateOne({ _id: job._id }, { $set: { total } }, function (err) { - if (err) { - self.apos.util.error(err); - } - }); + async setTotal(job, total) { + try { + await self.db.updateOne({ _id: job._id }, { $set: { total } }); + } catch (err) { + self.apos.util.error(err); + } }, // Mark the given job as ended. If `success` // is true the job is reported as an overall diff --git a/packages/apostrophe/modules/@apostrophecms/oembed/index.js b/packages/apostrophe/modules/@apostrophecms/oembed/index.js index d21e458da7..f76bfe5ea7 100644 --- a/packages/apostrophe/modules/@apostrophecms/oembed/index.js +++ b/packages/apostrophe/modules/@apostrophecms/oembed/index.js @@ -54,7 +54,8 @@ module.exports = { self.oembetter.allowlist(minimumAllowlist.concat(self.options.allowlist || [])); - const minimumEndpoints = self.options.minimumEndpoints || self.oembetter.suggestedEndpoints; + const minimumEndpoints = self.options.minimumEndpoints || + self.oembetter.suggestedEndpoints; self.oembetter.endpoints( minimumEndpoints.concat(self.options.endpoints || []) ); diff --git a/packages/apostrophe/modules/@apostrophecms/piece-page-type/index.js b/packages/apostrophe/modules/@apostrophecms/piece-page-type/index.js index 604e5753dd..b847966159 100644 --- a/packages/apostrophe/modules/@apostrophecms/piece-page-type/index.js +++ b/packages/apostrophe/modules/@apostrophecms/piece-page-type/index.js @@ -427,7 +427,14 @@ module.exports = { return metadata; } const [ pm ] = metadata; + // indexQuery is designed to be called with the + // index page in question as req.data.page. To + // reuse it for URL metadata purposes we must + // meet that expectation + const pageWas = req.data.page; + req.data.page = doc; const query = self.indexQuery(req); + req.data.page = pageWas; const filters = await self.getFiltersWithChoices(query, { allCounts: true }); // 1. Enumerate every filter + choice combination diff --git a/packages/apostrophe/modules/@apostrophecms/schema/ui/apos/components/AposArrayEditor.vue b/packages/apostrophe/modules/@apostrophecms/schema/ui/apos/components/AposArrayEditor.vue index 52e67b9735..0fa812820a 100644 --- a/packages/apostrophe/modules/@apostrophecms/schema/ui/apos/components/AposArrayEditor.vue +++ b/packages/apostrophe/modules/@apostrophecms/schema/ui/apos/components/AposArrayEditor.vue @@ -58,6 +58,7 @@ class="apos-modal-array-items__items" :selected="currentId" :model-value="withLabels(next)" + :draggable="field.draggable !== false" @update:model-value="update" @select="select" /> diff --git a/packages/apostrophe/modules/@apostrophecms/template/index.js b/packages/apostrophe/modules/@apostrophecms/template/index.js index a11a00e375..925a25932c 100644 --- a/packages/apostrophe/modules/@apostrophecms/template/index.js +++ b/packages/apostrophe/modules/@apostrophecms/template/index.js @@ -76,6 +76,32 @@ module.exports = { self.insertions = {}; self.runtimeNodes = {}; + // Install the .jsx require hook and teach the JSX runtime about + // Nunjucks' SafeString class so its instances pass through unescaped. + self.initJsx(); + + // Wire up the view-folder watcher with the two default invalidation + // handlers — Nunjucks loader caches and compiled .jsx modules. Both + // engines share a single set of chokidar watchers so we don't pay + // twice for watching the same directories. + const jsxLoader = require('./lib/jsxLoader.js'); + self.onViewChange(function clearNunjucksLoaderCaches() { + // Setting `cache = {}` mirrors the historical in-loader behavior + // and is exactly what Nunjucks itself reads when looking up a + // previously-loaded template. + for (const loader of Object.values(self.loaders || {})) { + loader.cache = {}; + } + }); + self.onViewChange(function invalidateJsxModules(filePath) { + if (filePath && filePath.endsWith('.jsx')) { + jsxLoader.invalidate(path.resolve(filePath)); + } else { + // Anything else (e.g. a Nunjucks file) might be a template imported + // by a `.jsx` file via require()/import — be safe and drop them all. + jsxLoader.invalidateAll(); + } + }); }, handlers(self) { return { @@ -117,9 +143,15 @@ module.exports = { }, 'apostrophe:destroy': { async nunjucksLoaderCleanup() { + // Older code paths used to manage chokidar watchers per loader; + // a no-op `destroy()` is still defined for backwards compat. for (const loader of Object.values(self.loaders || {})) { await loader.destroy(); } + }, + async closeViewWatchers() { + // Tear down chokidar watchers (Nunjucks + JSX share these). + await self.closeViewWatchers(); } } }; @@ -127,6 +159,19 @@ module.exports = { methods(self) { return { ...require('./lib/bundlesLoader')(self), + ...require('./lib/jsxRender')(self), + ...require('./lib/viewWatcher')(self), + + // Arm chokidar for the view-folder chain of the module whose views + // actually contain the resolved JSX file. For a same-module render + // that's the caller; for a cross-module render like + // `@apostrophecms/page` rendering `@apostrophecms/home-page:page` + // it's the target module. Idempotent per absolute directory. + watchJsxRenderTargets(callerModule, resolved) { + const owner = (resolved && self.apos.modules[resolved.moduleName]) || + callerModule; + self.watchViewFolders(self.getViewFolders(owner)); + }, // Add helpers in the namespace for a particular module. // They will be visible in nunjucks at @@ -261,6 +306,24 @@ module.exports = { let result; + // For named files, resolve through the module's view-folder + // chain. Chain position wins: a closer directory's .html/.njk + // beats a more distant directory's .jsx. JSX only takes + // precedence over Nunjucks within the same directory. See + // resolveTemplate. Falling back to Nunjucks happens automatically + // below when the resolved file is not JSX. + if (type === 'file') { + const resolved = self.resolveTemplate(module, s); + if (resolved && resolved.kind === 'jsx') { + const renderData = self.getRenderDataArgs(req, data, module); + result = await self.renderJsxTemplate(req, resolved, renderData, module); + if (process.platform === 'win32') { + result = result.replaceAll('\r', ''); + } + return result; + } + } + const args = self.getRenderArgs(req, data, module); const env = self.getEnv(req, module); @@ -495,6 +558,9 @@ module.exports = { } if (!self.loaders[key]) { self.loaders[key] = self.newLoader(moduleName, dirs); + // Register these dirs with the shared view watcher (idempotent + // per absolute path, so calling it for every loader is fine). + self.watchViewFolders(dirs); } return self.loaders[key]; }, @@ -1231,7 +1297,7 @@ module.exports = { async annotateDataForExternalFront(req, template, data, moduleName) { const docs = self.getDocsForExternalFront(req, template, data, moduleName); for (const doc of docs) { - self.annotateDocForExternalFront(doc, { scene: req.scene }); + await self.annotateDocForExternalFront(doc, { scene: req.scene }); } data.aposBodyData = await self.getBodyData(req); // Already contains module name too @@ -1283,16 +1349,31 @@ module.exports = { ].filter(doc => !!doc); }, - annotateDocForExternalFront(doc, { scene } = {}) { + async annotateDocForExternalFront(doc, { scene } = {}) { + const handled = new WeakSet(); + const missingAreas = []; self.apos.doc.walk(doc, (o, k, v) => { + if (o._edit === true && !handled.has(o)) { + handled.add(o); + for (const field of self.missingSchemaAreas(o)) { + missingAreas.push([ o, field ]); + } + } if (v && v.metaType === 'area') { - const manager = self.apos.util.getManagerOf(o); + // A missing manager here is expected (e.g. an area reached on a + // container without a manager) and handled below, so suppress the + // low-level per-call log and rely on the once-per-process warning. + const manager = self.apos.util.getManagerOf(o, { log: false }); if (!manager) { - self.apos.util.warnDevOnce('noManagerForDocInExternalFront', `No manager for: ${o.metaType} ${o.type || ''}`); + self.apos.util.warnDevOnce( + 'noManagerForDocInExternalFront', + `No manager for: ${o.metaType} ${o.type || ''}` + ); return; } const field = manager.schema.find(f => f.name === k); if (!field) { + v._isOrphan = true; self.apos.util.warnDevOnce( 'noSchemaFieldForAreaInExternalFront', `Area ${k} has no matching schema field in ${o.metaType} ${o.type || ''}` @@ -1302,6 +1383,14 @@ module.exports = { return self.annotateAreaForExternalFront(field, v, { scene }); } }); + // Materialize every missing area, after the walk so we never add keys + // to an object while it is being traversed. + for (const [ o, field ] of missingAreas) { + const area = await self.apos.area.addMissingArea(o, field.name); + area._edit = true; + area._docId = o._docId ?? (o.metaType === 'doc' ? o._id : null); + self.annotateAreaForExternalFront(field, area, { scene }); + } }, // Annotate an area for easy rendering by an external front end @@ -1310,6 +1399,7 @@ module.exports = { // at least as an empty array. annotateAreaForExternalFront(field, area, { scene } = {}) { + area._aposAnnotated = true; area.field = field; area.options = field.options; // Really widget configurations, but the method name is already set in @@ -1324,25 +1414,41 @@ module.exports = { }; }).filter(choice => !!choice); - area.items ||= []; + // Drop corrupt items (null, or not a widget). + area.items = (area.items || []).filter((item) => { + const valid = item && item.metaType === 'widget' && item.type; + if (!valid) { + self.apos.util.warnDevOnce( + 'corruptAreaItemInExternalFront', + `Dropping malformed item in area ${area._id || ''}` + ); + } + return valid; + }); + for (const item of area.items) { // Add _docId if area has one if (area._docId) { item._docId = area._docId; } - // Annotate each individual widget with its options - // Each widget must elect into this by creating an - // `annotateWidgetForExternalFront() method. + // Annotate each individual widget with its options. Each widget must + // elect into this by creating an `annotateWidgetForExternalFront()` + // method. const manager = self.apos.area.getWidgetManager(item.type); if (manager) { - const widgetOptions = manager.annotateWidgetForExternalFront(item, { scene }); - item._options = widgetOptions; + item._options = manager.annotateWidgetForExternalFront(item, { scene }); } else { self.apos.area.warnMissingWidgetType(item.type); - throw self.apos.error('invalid', 'Missing widget type'); } } + }, + + // The schema area fields of `object` that have no value yet. Returns an + // empty array for anything without a schema manager. + missingSchemaAreas(object) { + const schema = self.apos.util.getManagerOf(object, { log: false })?.schema ?? []; + return schema.filter(field => field.type === 'area' && !object[field.name]); } }; } diff --git a/packages/apostrophe/modules/@apostrophecms/template/lib/jsxLoader.js b/packages/apostrophe/modules/@apostrophecms/template/lib/jsxLoader.js new file mode 100644 index 0000000000..cf5fdfc12d --- /dev/null +++ b/packages/apostrophe/modules/@apostrophecms/template/lib/jsxLoader.js @@ -0,0 +1,128 @@ +// Compiles Apostrophe `.jsx` template files via Babel and registers a +// `require.extensions['.jsx']` hook so they can be loaded with `require()` +// (and `import` after CommonJS transformation) just like normal modules. +// +// Each compiled module is automatically prefixed with a `require()` of our +// JSX runtime so the `h` and `Fragment` identifiers produced by the Babel +// transform resolve without the user importing them. Both `import` and +// `require` work inside `.jsx` files because the CommonJS transform also +// runs. +// +// Source maps are kept in memory and wired through `source-map-support`, +// which means stack traces from a JSX template point at the original +// `views/page.jsx` line/column rather than the compiled output. + +const fs = require('fs'); +const Module = require('module'); +const babel = require('@babel/core'); +const sourceMapSupport = require('source-map-support'); + +const runtimePath = require.resolve('./jsxRuntime.js'); + +const sourceMaps = new Map(); +let installed = false; + +// Idempotent: register the require hook + source-map handler once per +// process even if multiple Apostrophe instances boot in the same Node +// process (e.g. tests, multisite). +function install() { + if (installed) { + return; + } + installed = true; + + sourceMapSupport.install({ + environment: 'node', + hookRequire: false, + handleUncaughtExceptions: false, + retrieveSourceMap(filename) { + const map = sourceMaps.get(filename); + if (!map) { + return null; + } + return { + url: filename, + map + }; + } + }); + + Module._extensions['.jsx'] = function(module, filename) { + const src = fs.readFileSync(filename, 'utf-8'); + const compiled = compile(src, filename); + sourceMaps.set(filename, compiled.map); + module._compile(compiled.code, filename); + }; +} + +// Compile a JSX source string for `filename`. Returns `{ code, map }`. +// `code` is CommonJS-compatible JS with our runtime injected at the top. +function compile(src, filename) { + let result; + try { + result = babel.transformSync(src, { + filename, + sourceMaps: true, + sourceFileName: filename, + babelrc: false, + configFile: false, + compact: false, + plugins: [ + [ + require.resolve('@babel/plugin-transform-react-jsx'), + { + pragma: '__aposJsx.h', + pragmaFrag: '__aposJsx.Fragment', + useBuiltIns: false, + throwIfNamespace: false + } + ], + require.resolve('@babel/plugin-transform-modules-commonjs') + ] + }); + } catch (e) { + // Babel errors already include code frames pointing at the offending + // line/column. Preserve that detail and add the file path for clarity. + const err = new Error(`JSX compile error in ${filename}: ${e.message}`); + err.cause = e; + err.code = 'APOS_JSX_COMPILE_ERROR'; + err.filename = filename; + throw err; + } + + // Inject runtime references. Using a single `__aposJsx` namespace avoids + // colliding with user variables named `h` or `Fragment` while still + // matching the pragma we passed to Babel above. Source maps remain valid + // because we only prepend a single line and rely on a leading `\n` to + // keep line numbers stable. + const prefix = `var __aposJsx = require(${JSON.stringify(runtimePath)});\n`; + return { + code: prefix + result.code, + map: result.map + }; +} + +// Drop a single .jsx file from the require cache and our source-map cache. +// Called by the template module's chokidar watcher when a JSX file changes, +// so the next render picks up the new code without restarting the process. +function invalidate(filename) { + sourceMaps.delete(filename); + delete Module._cache[filename]; +} + +// Drop every cached .jsx module and source map. Used when watcher events +// don't carry a specific path or when an unknown view file was modified. +function invalidateAll() { + for (const filename of sourceMaps.keys()) { + delete Module._cache[filename]; + } + sourceMaps.clear(); +} + +module.exports = { + install, + compile, + invalidate, + invalidateAll, + runtimePath +}; diff --git a/packages/apostrophe/modules/@apostrophecms/template/lib/jsxRender.js b/packages/apostrophe/modules/@apostrophecms/template/lib/jsxRender.js new file mode 100644 index 0000000000..f1320fe20a --- /dev/null +++ b/packages/apostrophe/modules/@apostrophecms/template/lib/jsxRender.js @@ -0,0 +1,490 @@ +// JSX render orchestration. Mixed into the `@apostrophecms/template` +// module by `index.js`, this file implements: +// +// * Template path resolution that walks the same module chain as Nunjucks +// but knows about `.jsx` and prefers it when present alongside `.html`. +// * `renderJsxTemplate(req, resolved, data, module)` which loads the +// compiled JSX module, invokes its default-exported function with +// `(data, helpers)`, and flattens the resulting node tree into HTML. +// * The `Area`, `Component`, `Template`, `Extend`, and `Widget` runtime +// helpers exposed to JSX templates as the second argument to their +// default function. +// * The cross-engine bridge: when a JSX template invokes `Template`/ +// `Extend` against a Nunjucks `.html` layout, the helper synthesizes a +// Nunjucks string that extends the target and turns each named prop +// into a `{% block ... %}` override. + +const fs = require('fs'); +const path = require('path'); +const _ = require('lodash'); + +const jsxLoader = require('./jsxLoader.js'); +const { + Raw, flatten, registerSafeClass +} = require('./jsxRuntime.js'); + +const TEMPLATE_EXTENSIONS = [ 'jsx', 'njk', 'html' ]; + +module.exports = function(self) { + return { + // Install the global JSX `require` hook and teach the runtime about + // Nunjucks `SafeString` instances so they pass through unescaped. + initJsx() { + jsxLoader.install(); + registerSafeClass(self.nunjucks.runtime.SafeString); + }, + + // Walk a module's view-folder chain and find the first file matching + // `name` with one of the supported extensions. JSX wins over Nunjucks + // when both exist in the same directory, but the chain ordering still + // takes precedence (a child module's `.html` still overrides a parent + // module's `.jsx` if it appears earlier in the chain). + // + // `name` may be `'localname'` (resolved against the supplied + // `module`'s chain) or `'modulename:localname'` (resolved against the + // named module's chain). An explicit extension is honored verbatim. + // + // Returns `{ kind: 'jsx' | 'nunjucks', path, ext, moduleName, + // baseName, ext, requestedName }` or `null` when nothing was found. + resolveTemplate(module, name) { + let moduleName = module.__meta.name; + let filename = name; + const colonAt = name.indexOf(':'); + if (colonAt !== -1) { + moduleName = name.substring(0, colonAt); + filename = name.substring(colonAt + 1); + } + const targetModule = self.apos.modules[moduleName]; + if (!targetModule) { + return null; + } + const dirs = self.getViewFolders(targetModule); + const m = filename.match(/^(.*)\.([^/.]+)$/); + let baseName = filename; + let requestedExt = null; + if (m) { + baseName = m[1]; + requestedExt = m[2]; + } + // For requests without an extension, or for the well-known template + // extensions, allow falling back to any of them. For an unknown + // extension (e.g. `.svg`) preserve current Nunjucks-loader behavior: + // try only the literal name. + const exts = (!requestedExt || TEMPLATE_EXTENSIONS.includes(requestedExt)) + ? TEMPLATE_EXTENSIONS + : [ requestedExt ]; + + for (const dir of dirs) { + for (const ext of exts) { + const fullpath = path.join(dir, `${baseName}.${ext}`); + if (fs.existsSync(fullpath)) { + return { + kind: ext === 'jsx' ? 'jsx' : 'nunjucks', + path: fullpath, + ext, + moduleName, + baseName, + relativeName: `${baseName}.${ext}`, + requestedName: name + }; + } + } + } + return null; + }, + + // Render the JSX module at `resolved.path` against `data`. Loads the + // compiled module via Node's require (the `.jsx` extension hook + // installed by `jsxLoader` does the Babel transform on first load), + // calls its default function, and flattens the returned node tree. + async renderJsxTemplate(req, resolved, data, module) { + self.watchJsxRenderTargets(module, resolved); + let mod; + try { + mod = require(resolved.path); + } catch (e) { + throw decorateJsxError(e, resolved.path); + } + const fn = (mod && mod.default) || mod; + if (typeof fn !== 'function') { + throw new Error( + `JSX template ${resolved.path} must export a default function. ` + + `Got ${typeof fn}.` + ); + } + const helpers = self.buildJsxHelpers(req, module, data); + let result; + try { + result = await fn(data, helpers); + } catch (e) { + throw decorateJsxError(e, resolved.path); + } + try { + return await flatten(result); + } catch (e) { + throw decorateJsxError(e, resolved.path); + } + }, + + // Build the `{ apos, helpers, Area, Component, Extend, Template, + // Widget, ... }` object passed as the second argument to every JSX + // template. The closure captures `req`, `module` (the module whose + // template is currently being rendered, used to resolve `Template` + // names without a `module:` prefix), and `ambientData` (the merged + // render data for the current invocation, threaded through + // `Template`/`Extend` to mirror Nunjucks' `extends` semantics). + buildJsxHelpers(req, module, ambientData) { + const helpers = { + // Full `self.apos`. Unlike Nunjucks, JSX supports await, so + // there is no need to restrict access to this object. + apos: self.apos, + // The Nunjucks-compatible wrapper. Carries `addHelpers`-registered + // helpers under `helpers.modules['module-name'].method(...)` and + // module aliases, matching the `apos` object available in + // Nunjucks templates. Distinct from the JSX `apos` above. + helpers: self.templateApos, + // Localization helper, matching the Nunjucks `__t` global. + __t: req.t && req.t.bind(req), + + Area: (props) => self.jsxArea(req, props), + Component: (props) => self.jsxComponent(req, props), + Widget: (props) => self.jsxWidget(req, props), + Template: (props) => self.jsxInvoke(req, module, props, { + mode: 'include', + ambientData + }), + Extend: (props) => self.jsxInvoke(req, module, props, { + mode: 'extend', + ambientData + }) + }; + return helpers; + }, + + // Implementation of the `` helper. + // Mirrors the Nunjucks `{% area %}` custom tag closely so behavior + // (including stub-area persistence) is identical. + jsxArea(req, props) { + const { + doc, name, with: ctx + } = props || {}; + return (async () => { + if (!doc || typeof doc !== 'object') { + throw new Error( + 'Area: the `doc` prop must be an existing doc or widget object.' + ); + } + if (typeof name !== 'string') { + throw new Error('Area: the `name` prop must be a string.'); + } + if (!name.match(/^\w+$/)) { + throw new Error( + 'Area: area names must consist only of letters, digits, and underscores.' + ); + } + let area = doc[name]; + if (!area) { + // Same stub-into-db logic as the {% area %} tag, so that newly + // added schema fields get a persistent `_id` on first render. + area = { + metaType: 'area', + _id: self.apos.util.generateId(), + items: [] + }; + doc[name] = area; + const docId = doc._docId || ((doc.metaType === 'doc') ? doc._id : null); + if (docId) { + let mainDoc = await self.apos.doc.db.findOne({ _id: docId }); + if (!mainDoc) { + throw self.apos.error('notfound'); + } + let docDotPath; + try { + docDotPath = (doc._id === docId) + ? '' + : self.apos.util.findNestedObjectAndDotPathById(mainDoc, doc._id).dotPath; + } catch (e) { + throw self.apos.error('notfound'); + } + const areaDotPath = docDotPath ? `${docDotPath}.${name}` : name; + await self.apos.doc.db.updateOne({ + _id: docId, + [areaDotPath]: { $eq: null } + }, { + $set: { + [areaDotPath]: self.apos.util.clonePermanent(area) + } + }); + mainDoc = await self.apos.doc.db.findOne({ _id: docId }); + area._id = self.apos.util.get(mainDoc, areaDotPath)._id; + } + } + const manager = self.apos.util.getManagerOf(doc); + const field = manager && manager.schema.find(f => f.name === name); + if (!field) { + throw new Error( + `Area: the doc of type ${doc.type} with the slug ${doc.slug} ` + + `has no field named ${name}.` + ); + } + area._fieldId = field._id; + area._docId = doc._docId || ((doc.metaType === 'doc') ? doc._id : null); + area._edit = area._edit || doc._edit; + self.apos.area.prepForRender(area, doc, name); + const html = await self.apos.area.renderArea(req, area, ctx); + return new Raw(html); + })(); + }, + + // Implementation of ``. + // Looks up the named async component, awaits it, then renders the + // component's matching template via the existing module render path. + jsxComponent(req, props) { + const { + module: moduleName, name, children, ...rest + } = props || {}; + return (async () => { + if (typeof moduleName !== 'string' || typeof name !== 'string') { + throw new Error( + 'Component: both `module` and `name` props must be strings.' + ); + } + const target = self.apos.modules[moduleName]; + if (!target) { + throw new Error( + `Component: module "${moduleName}" does not exist. ` + + 'It must be a real, instantiated module, not a base class.' + ); + } + if (!(target.components && target.components[name])) { + throw new Error( + `Component: ${moduleName}:${name} is not a registered async component.` + ); + } + // Components receive plain data: any JSX nodes passed as props + // (including `children`) need to be flattened to HTML strings + // first so the underlying Nunjucks/JSX component template can + // safely render them. + const inputProps = await self.flattenJsxProps({ + ...rest, + children + }); + const result = await self.apos.util.recursionGuard( + req, + `component:${moduleName}:${name}`, + async () => { + const input = await target.components[name](req, inputProps); + return target.render(req, name, input); + } + ); + if (result === undefined) { + // Recursion guard kicked in. + return new Raw(''); + } + return new Raw(result); + })(); + }, + + // Implementation of ``. + // Mirrors the Nunjucks `{% widget %}` tag, intended only for users + // reimplementing `area.html` in JSX. + jsxWidget(req, props) { + const { + widget, options, with: contextOptions + } = props || {}; + return (async () => { + if (!widget) { + self.apos.util.warn('a null widget was encountered.'); + return new Raw(''); + } + const opts = options || {}; + let ctxOpts = {}; + if (contextOptions && typeof contextOptions === 'object' && contextOptions[widget.type]) { + ctxOpts = (typeof contextOptions[widget.type] === 'object') + ? contextOptions[widget.type] + : {}; + } + const manager = self.apos.area.getWidgetManager(widget.type); + if (!manager) { + self.apos.area.warnMissingWidgetType(widget.type); + return new Raw(''); + } + const html = await manager.output(req, widget, opts, ctxOpts); + return new Raw(html); + })(); + }, + + // Shared implementation of `Template` and `Extend`. The rules: + // + // * Strip `templateName`/`name` per the spec — `templateName` always + // wins as the file selector, otherwise `name` is the file selector + // AND is *not* passed through as a data prop. + // * Resolve the file. JSX targets receive the props (plus ambient + // data) and `children` natively; Nunjucks targets receive props + // as block overrides (`extend` mode) or as data (`include` mode + // when called via `Template` against a Nunjucks file). + jsxInvoke(req, callerModule, props, { mode, ambientData }) { + const { + templateName, name, ...rest + } = props || {}; + let targetName; + const dataProps = { ...rest }; + if (templateName !== undefined) { + targetName = templateName; + // Per spec: `name` is forwarded as a normal prop only when + // `templateName` is also present. + if (name !== undefined) { + dataProps.name = name; + } + } else { + targetName = name; + } + if (typeof targetName !== 'string') { + throw new Error( + 'Template/Extend: pass a string to the `templateName` prop ' + + '(or `name` when no other prop named `name` is needed).' + ); + } + return (async () => { + const resolved = self.resolveTemplate(callerModule, targetName); + if (!resolved) { + throw new Error(`Template/Extend: could not resolve template "${targetName}".`); + } + if (resolved.kind === 'jsx') { + // For JSX targets both Template and Extend behave the same: + // the props (with the parent's ambient data underneath) are + // passed straight through. JSX values like `children` flow as + // node trees — they are flattened only when the target template + // emits them. + const targetModule = self.apos.modules[resolved.moduleName]; + const merged = { + ...(ambientData || {}), + ...dataProps + }; + const html = await self.renderJsxTemplate(req, resolved, merged, targetModule); + return new Raw(html); + } + // Nunjucks target. + if (mode === 'extend') { + return self.renderNunjucksWithBlocks( + req, resolved, dataProps, ambientData + ); + } + // mode === 'include': render the Nunjucks template with our + // props merged on top of ambient data, the same way a Nunjucks + // `{% include %}` would inherit data. + const flatProps = await self.flattenJsxProps(dataProps); + const targetModule = self.apos.modules[resolved.moduleName]; + const data = { + ...(ambientData || {}), + ...flatProps + }; + const html = await targetModule.render(req, resolved.relativeName, data); + return new Raw(html); + })(); + }, + + // Invoke a Nunjucks template via `extends` so that JSX-supplied props + // become `{% block %}` overrides. The synthetic Nunjucks template + // declares one block per prop, each emitting the matching string + // marked safe so already-rendered HTML survives. + async renderNunjucksWithBlocks(req, resolved, props, ambientData) { + const targetModule = self.apos.modules[resolved.moduleName]; + const blocks = {}; + for (const key of Object.keys(props)) { + const html = await flattenToHtml(props[key]); + blocks[key] = html; + } + const blockNames = Object.keys(blocks); + const targetRef = `${resolved.moduleName}:${resolved.relativeName}`; + const lines = [ `{% extends ${JSON.stringify(targetRef)} %}` ]; + for (const blockName of blockNames) { + if (!/^[A-Za-z_][\w]*$/.test(blockName)) { + throw new Error( + 'Template/Extend: prop names used as block overrides must be ' + + `valid identifiers (got "${blockName}").` + ); + } + lines.push( + `{% block ${blockName} %}{{ data.aposJsxBlocks[${JSON.stringify(blockName)}] | safe }}{% endblock %}` + ); + } + const synthetic = lines.join('\n'); + // Layer ambient page data underneath so the Nunjucks layout has + // access to `data.outerLayout`, `data.page`, etc.; our blocks + // override only what they explicitly name. + const data = { + ...(ambientData || {}), + aposJsxBlocks: blocks + }; + const html = await targetModule.renderString(req, synthetic, data); + return new Raw(html); + }, + + // Walk a props object, flattening any JSX node values to HTML strings + // so they can cross into Nunjucks (which can't deal with our internal + // node arrays). Plain primitive props pass through unchanged. + async flattenJsxProps(props) { + const out = {}; + for (const key of Object.keys(props)) { + const value = props[key]; + if (value === undefined) { + continue; + } + if (isJsxNode(value)) { + out[key] = self.safe(await flatten(value)); + } else { + out[key] = value; + } + } + return out; + } + }; +}; + +// Decide whether a value should be flattened by the JSX runtime before +// being handed to Nunjucks. Arrays, Raw markers, and thenables produced +// by our helpers all need flattening; everything else (strings, numbers, +// docs, options objects) is forwarded as-is. +function isJsxNode(value) { + if (value == null) { + return false; + } + if (Array.isArray(value)) { + return true; + } + if (value instanceof Raw) { + return true; + } + if (typeof value === 'object' && typeof value.then === 'function') { + return true; + } + return false; +} + +// Convenience wrapper used when a JSX-supplied prop becomes a Nunjucks +// block override: regardless of input type, produce the final HTML string +// that the synthesized template will emit via the `safe` filter. +async function flattenToHtml(value) { + if (value == null || value === false) { + return ''; + } + if (isJsxNode(value)) { + return await flatten(value); + } + return String(value); +} + +// Annotate exceptions thrown out of a JSX template with the file path so +// the error log clearly identifies which template failed. Source maps +// (installed by `jsxLoader`) take care of accurate line/column info. +function decorateJsxError(error, file) { + if (!error || typeof error !== 'object') { + return error; + } + if (!error.aposJsxFile) { + error.aposJsxFile = file; + error.message = `[JSX template ${file}] ${error.message}`; + } + return error; +} diff --git a/packages/apostrophe/modules/@apostrophecms/template/lib/jsxRuntime.js b/packages/apostrophe/modules/@apostrophecms/template/lib/jsxRuntime.js new file mode 100644 index 0000000000..915a11900c --- /dev/null +++ b/packages/apostrophe/modules/@apostrophecms/template/lib/jsxRuntime.js @@ -0,0 +1,276 @@ +// JSX runtime used by Apostrophe `.jsx` templates. The classic Babel +// transform (`@babel/plugin-transform-react-jsx`) compiles `...` +// into `h(Tag, { a: x }, ...children)` and `<>...` into +// `h(Fragment, null, ...children)`. The `h` function and `Fragment` symbol +// are injected into every compiled module by `jsxLoader.js`. +// +// Output model: `h` returns a nested array of strings, `Raw` markers, +// promises, and arrays. The array is flattened by `flatten()` which awaits +// every promise, escapes plain string values, and joins everything into a +// final HTML string. This array+promise model is what allows JSX templates +// to call asynchronous helpers (`Area`, `Component`, `Template`, `Extend`) +// directly inside markup without wrapping them in `await`. + +const voidElements = require('void-elements'); + +const Fragment = Symbol('AposJsxFragment'); + +// Marker class for already-rendered raw HTML. Anything wrapped in a `Raw` +// is emitted into the final output without any additional escaping. +class Raw { + constructor(html) { + this.html = (html == null) ? '' : String(html); + } +} + +// Compatibility hook: Nunjucks `SafeString` instances (returned by Apostrophe +// helpers like `apos.area.html()` and the `safe` filter) need to flow through +// JSX templates without being escaped a second time. `template/index.js` +// registers Nunjucks's `SafeString` class via `registerSafeClass()`; any +// instance of a registered class is treated as raw HTML. +const safeClasses = []; + +function registerSafeClass(cls) { + if (cls && !safeClasses.includes(cls)) { + safeClasses.push(cls); + } +} + +function isSafeInstance(value) { + if (value == null || typeof value !== 'object') { + return false; + } + for (const cls of safeClasses) { + if (value instanceof cls) { + return true; + } + } + return false; +} + +function escapeHtml(value) { + return String(value) + .replace(/&/g, '&') + .replace(//g, '>'); +} + +function escapeAttr(value) { + return String(value) + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(//g, '>'); +} + +// Build the JSX node for an element, function component, or fragment. +// For function types (including the runtime helpers `Area`, `Component`, +// `Template`, `Extend`, `Widget`) we invoke the function with a single +// `props` object that includes `children`, matching React conventions. +function h(type, props, ...children) { + props = props || {}; + if (type === Fragment) { + return children; + } + if (typeof type === 'function') { + const finalProps = { ...props }; + if (children.length > 0) { + finalProps.children = (children.length === 1) ? children[0] : children; + } + return type(finalProps); + } + if (typeof type !== 'string') { + throw new Error(`Invalid JSX element type: ${typeof type}`); + } + return buildElement(type, props, children); +} + +function buildElement(tag, props, children) { + let attrs = ''; + let dangerous = null; + for (const key of Object.keys(props)) { + if (key === 'children' || key === 'key' || key === 'ref') { + continue; + } + if (key === 'dangerouslySetInnerHTML') { + const v = props[key]; + if (v && typeof v.__html === 'string') { + dangerous = v.__html; + } else if (v && v.__html != null) { + dangerous = String(v.__html); + } + continue; + } + const value = props[key]; + if (value == null || value === false) { + continue; + } + const attrName = jsxAttrName(key); + if (value === true) { + attrs += ` ${attrName}`; + } else { + attrs += ` ${attrName}="${escapeAttr(value)}"`; + } + } + + if (dangerous != null) { + return [ + new Raw(`<${tag}${attrs}>`), + new Raw(dangerous), + new Raw(``) + ]; + } + + if (voidElements[tag] && children.length === 0) { + return [ new Raw(`<${tag}${attrs} />`) ]; + } + + return [ + new Raw(`<${tag}${attrs}>`), + ...children, + new Raw(``) + ]; +} + +// Match React's friendly attribute names to standard HTML/SVG attribute +// names. `data-*` and `aria-*` props pass through verbatim; SVG presentation +// attributes (`strokeWidth`, `fillRule`, etc.) are converted to the +// kebab-case form actually understood by browsers when the document is +// parsed as text/html. The handful of SVG attributes that are *natively* +// camelCase (e.g. `viewBox`, `preserveAspectRatio`) stay as-is. +const svgAttrMap = { + // Stroke + strokeWidth: 'stroke-width', + strokeLinecap: 'stroke-linecap', + strokeLinejoin: 'stroke-linejoin', + strokeDasharray: 'stroke-dasharray', + strokeDashoffset: 'stroke-dashoffset', + strokeMiterlimit: 'stroke-miterlimit', + strokeOpacity: 'stroke-opacity', + // Fill + fillOpacity: 'fill-opacity', + fillRule: 'fill-rule', + // Clip / mask + clipPath: 'clip-path', + clipRule: 'clip-rule', + // Text + textAnchor: 'text-anchor', + textDecoration: 'text-decoration', + alignmentBaseline: 'alignment-baseline', + baselineShift: 'baseline-shift', + dominantBaseline: 'dominant-baseline', + fontFamily: 'font-family', + fontSize: 'font-size', + fontStyle: 'font-style', + fontVariant: 'font-variant', + fontWeight: 'font-weight', + letterSpacing: 'letter-spacing', + wordSpacing: 'word-spacing', + // Generic + colorInterpolation: 'color-interpolation', + colorInterpolationFilters: 'color-interpolation-filters', + colorProfile: 'color-profile', + colorRendering: 'color-rendering', + fillRendering: 'fill-rendering', + imageRendering: 'image-rendering', + shapeRendering: 'shape-rendering', + textRendering: 'text-rendering', + pointerEvents: 'pointer-events', + unicodeBidi: 'unicode-bidi', + vectorEffect: 'vector-effect', + writingMode: 'writing-mode', + enableBackground: 'enable-background', + floodColor: 'flood-color', + floodOpacity: 'flood-opacity', + glyphOrientationHorizontal: 'glyph-orientation-horizontal', + glyphOrientationVertical: 'glyph-orientation-vertical', + lightingColor: 'lighting-color', + markerEnd: 'marker-end', + markerMid: 'marker-mid', + markerStart: 'marker-start', + overflowWrap: 'overflow-wrap', + paintOrder: 'paint-order', + stopColor: 'stop-color', + stopOpacity: 'stop-opacity', + // Linked references + xlinkHref: 'xlink:href', + xlinkRole: 'xlink:role', + xlinkTitle: 'xlink:title', + xlinkType: 'xlink:type', + xlinkArcrole: 'xlink:arcrole', + xlinkActuate: 'xlink:actuate', + xlinkShow: 'xlink:show', + xmlBase: 'xml:base', + xmlLang: 'xml:lang', + xmlSpace: 'xml:space', + xmlnsXlink: 'xmlns:xlink' +}; + +function jsxAttrName(name) { + if (name === 'className') { + return 'class'; + } + if (name === 'htmlFor') { + return 'for'; + } + const svg = svgAttrMap[name]; + if (svg) { + return svg; + } + return name; +} + +// Walk the tree of strings, Raw markers, promises, and arrays, awaiting +// every promise and producing a final HTML string. Plain strings/numbers +// are escaped; Raw and Nunjucks SafeString values are not. +async function flatten(node) { + const out = []; + await walk(node, out); + return out.join(''); +} + +async function walk(node, out) { + if (node == null || node === false || node === true) { + return; + } + if (Array.isArray(node)) { + for (const item of node) { + await walk(item, out); + } + return; + } + if (node instanceof Raw) { + out.push(node.html); + return; + } + if (isSafeInstance(node)) { + out.push(node.toString()); + return; + } + if (typeof node === 'object' && typeof node.then === 'function') { + const resolved = await node; + await walk(resolved, out); + return; + } + if (typeof node === 'string') { + out.push(escapeHtml(node)); + return; + } + if (typeof node === 'number' || typeof node === 'bigint') { + out.push(escapeHtml(String(node))); + return; + } + // Anything else (objects without a known handler) is coerced to string + // and escaped, mirroring how React stringifies unexpected children. + out.push(escapeHtml(String(node))); +} + +module.exports = { + h, + Fragment, + Raw, + flatten, + escapeHtml, + escapeAttr, + registerSafeClass +}; diff --git a/packages/apostrophe/modules/@apostrophecms/template/lib/nunjucksLoader.js b/packages/apostrophe/modules/@apostrophecms/template/lib/nunjucksLoader.js index 3bfe99801b..38c0bd133f 100644 --- a/packages/apostrophe/modules/@apostrophecms/template/lib/nunjucksLoader.js +++ b/packages/apostrophe/modules/@apostrophecms/template/lib/nunjucksLoader.js @@ -7,23 +7,26 @@ // Note that if @apostrophecms/template has a project-level override // of outerLayout.html, that will be loaded instead. This is // intentional. +// +// File watching and cache invalidation are deliberately NOT handled here: +// the template module wires up `viewWatcher.js` once for both Nunjucks and +// JSX, and clears every loader's `cache` (read by Nunjucks itself) when a +// view file changes. const fs = require('fs'); const path = require('path'); const _ = require('lodash'); const { stripIndent } = require('common-tags'); -const chokidar = require('chokidar'); module.exports = function(moduleName, searchPaths, noWatch, templates, options) { const self = this; - self.watches = []; options = options || {}; const extensions = options.extensions || [ 'njk', 'html' ]; self.moduleName = moduleName; self.templates = templates; - self.init = function(searchPaths, noWatch) { + self.init = function(searchPaths) { self.pathsToNames = {}; if (searchPaths) { searchPaths = Array.isArray(searchPaths) ? searchPaths : [ searchPaths ]; @@ -32,34 +35,6 @@ module.exports = function(moduleName, searchPaths, noWatch, templates, options) } else { self.searchPaths = []; } - // Unless and until chokidar declares this a supported config, - // no watching in WSL (it doesn't work without chokidar either) - if ((!noWatch) && (!require('is-wsl'))) { - _.each(self.searchPaths, function(p) { - if (fs.existsSync(p)) { - try { - const watcher = chokidar.watch(p); - watcher.on('change', (path, stats) => { - // Just blow the whole cache if anything is modified. Much - // simpler, avoids several false negatives, and works well for a - // CMS in dev. -Tom - self.cache = {}; - }); - self.watches.push(watcher); - } catch (e) { - if (!self.firstWatchFailure) { - // Don't crash in broken environments (not sure if any are left - // thanks to chokidar, but still a useful warning to have if it - // comes up) - self.firstWatchFailure = true; - self.templates.apos.util.warn('WARNING: fs.watch does not work on this system. That is OK but you\n' + - 'will have to restart to see any template changes take effect.'); - } - self.templates.apos.util.error(e); - } - } - }); - } }; self.isRelative = function(filename) { @@ -183,11 +158,11 @@ module.exports = function(moduleName, searchPaths, noWatch, templates, options) } }; - self.destroy = async () => { - for (const watch of self.watches) { - await watch.close(); - } - }; + // Retained for backwards compatibility — file watching now lives in + // `viewWatcher.js`, owned by the template module itself, so there is + // nothing for the loader to dispose of. Callers (Apostrophe internals + // and tests) may still invoke `destroy()` and that must keep working. + self.destroy = async () => {}; self.init(searchPaths, noWatch); }; diff --git a/packages/apostrophe/modules/@apostrophecms/template/lib/viewWatcher.js b/packages/apostrophe/modules/@apostrophecms/template/lib/viewWatcher.js new file mode 100644 index 0000000000..3492b8021c --- /dev/null +++ b/packages/apostrophe/modules/@apostrophecms/template/lib/viewWatcher.js @@ -0,0 +1,113 @@ +// File-watching for view directories shared by Nunjucks and JSX. +// +// Owns one chokidar watcher per directory and dispatches change events to +// any number of registered handlers. The template module wires this up at +// init time with two handlers: clear every Nunjucks loader's cache, and +// invalidate compiled `.jsx` modules so the next render reloads them. +// +// WSL safety: chokidar's underlying file events are unreliable on WSL, so +// we deliberately skip watching there — same behavior as the previous +// in-loader code. Developers running on WSL will need to restart to pick +// up template edits, but the rest of Apostrophe still functions. +// +// Returns a methods mixin compatible with `@apostrophecms/module`'s +// `methods` factory pattern, the same way `bundlesLoader` and `jsxRender` +// are mixed in. + +const fs = require('fs'); +const chokidar = require('chokidar'); +const isWsl = require('is-wsl'); + +module.exports = (self) => { + // Lazily-allocated state. Lives on `self` so it survives across method + // calls and can be inspected by tests if needed. + if (!self._viewWatchers) { + self._viewWatchers = []; + } + if (!self._viewWatchedDirs) { + self._viewWatchedDirs = new Set(); + } + if (!self._viewChangeHandlers) { + self._viewChangeHandlers = []; + } + + return { + // Begin watching the supplied directories for change events. Idempotent + // per absolute path: subsequent calls with the same dir do not create + // duplicate watchers. Honors the WSL skip flag and the loader's + // `noWatch` option (templates module exposes this via + // `options.loader.noWatch`). + watchViewFolders(dirs) { + if (self.options.loader && self.options.loader.noWatch) { + return; + } + // In production the file tree is static and watching only burns + // file descriptors. Honor NODE_ENV unconditionally. + if (process.env.NODE_ENV === 'production') { + return; + } + // chokidar's recursive watching is not reliable on WSL; preserve the + // historical behavior of skipping watch setup entirely there. + if (isWsl) { + return; + } + for (const dir of dirs) { + if (self._viewWatchedDirs.has(dir)) { + continue; + } + if (!fs.existsSync(dir)) { + continue; + } + self._viewWatchedDirs.add(dir); + try { + const watcher = chokidar.watch(dir); + watcher.on('change', (filePath) => { + for (const handler of self._viewChangeHandlers) { + try { + handler(filePath); + } catch (e) { + self.apos.util.error(e); + } + } + }); + self._viewWatchers.push(watcher); + } catch (e) { + // Don't crash in broken environments. Warn at most once so the + // logs don't get spammed for every dir we fail to watch. + if (!self._viewFirstWatchFailure) { + self._viewFirstWatchFailure = true; + self.apos.util.warn( + 'WARNING: fs.watch does not work on this system. That is OK but you\n' + + 'will have to restart to see any template changes take effect.' + ); + } + self.apos.util.error(e); + } + } + }, + + // Register a callback invoked with the absolute path of the changed + // file every time chokidar reports a `change` event. Handlers run in + // registration order. Errors thrown by a handler are logged but never + // stop other handlers from running. + onViewChange(handler) { + self._viewChangeHandlers.push(handler); + }, + + // Tear down every chokidar watcher created via `watchViewFolders`. + // Called from the `apostrophe:destroy` handler so a process that + // builds and tears down multiple Apostrophe instances (tests, the + // multisite harness) doesn't leak file descriptors. + async closeViewWatchers() { + const watchers = self._viewWatchers.splice(0, self._viewWatchers.length); + self._viewWatchedDirs.clear(); + for (const watcher of watchers) { + try { + await watcher.close(); + } catch (e) { + self.apos.util.error(e); + } + } + } + }; +}; diff --git a/packages/apostrophe/modules/@apostrophecms/ui/ui/apos/components/AposSlat.vue b/packages/apostrophe/modules/@apostrophecms/ui/ui/apos/components/AposSlat.vue index 7d5ca444ce..4068a51a01 100644 --- a/packages/apostrophe/modules/@apostrophecms/ui/ui/apos/components/AposSlat.vue +++ b/packages/apostrophe/modules/@apostrophecms/ui/ui/apos/components/AposSlat.vue @@ -8,6 +8,7 @@ 'apos-is-only-child': slatCount === 1, 'apos-is-selected': selected, 'apos-is-disabled': disabled, + 'apos-is-not-draggable': !draggable, }" :aria-current="engaged" role="listitem" @@ -22,7 +23,7 @@ >
@@ -158,6 +159,10 @@ export default { editorIcon: { type: String, default: null + }, + draggable: { + type: Boolean, + default: true } }, emits: [ 'engage', 'disengage', 'move', 'remove', 'item-clicked', 'select' ], @@ -209,7 +214,7 @@ export default { return e.target.click(); }, toggleEngage() { - if (this.slatCount > 1) { + if (this.draggable && this.slatCount > 1) { if (this.engaged) { this.disengage(); } else { @@ -278,7 +283,8 @@ export default { } &.apos-slat-list__item--disabled, - &.apos-is-only-child { + &.apos-is-only-child, + &.apos-is-not-draggable { &:hover, &:active { cursor: default; diff --git a/packages/apostrophe/modules/@apostrophecms/ui/ui/apos/components/AposSlatList.vue b/packages/apostrophe/modules/@apostrophecms/ui/ui/apos/components/AposSlatList.vue index 81a9c9e35d..8e4d0f09ba 100644 --- a/packages/apostrophe/modules/@apostrophecms/ui/ui/apos/components/AposSlatList.vue +++ b/packages/apostrophe/modules/@apostrophecms/ui/ui/apos/components/AposSlatList.vue @@ -26,6 +26,7 @@ 'apos-input--error': duplicate }" :disabled="disabled" + :draggable="draggable" :engaged="engaged === item._id" :parent="listId" :slat-count="next.length" @@ -87,6 +88,10 @@ export default { duplicate: { type: String, default: null + }, + draggable: { + type: Boolean, + default: true } }, emits: [ 'item-clicked', 'select', 'update:modelValue' ], @@ -104,7 +109,7 @@ export default { dragOptions() { return { animation: 0, - disabled: this.disabled || this.next.length <= 1, + disabled: !this.draggable || this.disabled || this.next.length <= 1, ghostClass: 'apos-is-dragging' }; } diff --git a/packages/apostrophe/modules/@apostrophecms/ui/ui/apos/scss/global/_inputs.scss b/packages/apostrophe/modules/@apostrophecms/ui/ui/apos/scss/global/_inputs.scss index 60dcd1b43c..36d857504c 100644 --- a/packages/apostrophe/modules/@apostrophecms/ui/ui/apos/scss/global/_inputs.scss +++ b/packages/apostrophe/modules/@apostrophecms/ui/ui/apos/scss/global/_inputs.scss @@ -372,6 +372,8 @@ @include type-base; & { + // Do not remove - it fixes unintended `sr-only` side effect. + position: relative; display: flex; align-items: center; color: var(--a-base-2); diff --git a/packages/apostrophe/modules/@apostrophecms/uploadfs/index.js b/packages/apostrophe/modules/@apostrophecms/uploadfs/index.js index 9353fe4340..694d7df4a0 100644 --- a/packages/apostrophe/modules/@apostrophecms/uploadfs/index.js +++ b/packages/apostrophe/modules/@apostrophecms/uploadfs/index.js @@ -41,6 +41,9 @@ module.exports = { const uploadfsSettings = {}; _.merge(uploadfsSettings, uploadfsDefaultSettings); _.merge(uploadfsSettings, options); + if (process.env.APOS_UPLOADFS_DISABLED_FILE_KEY) { + uploadfsSettings.disabledFileKey = process.env.APOS_UPLOADFS_DISABLED_FILE_KEY; + } if (process.env.APOS_S3_BUCKET) { _.merge(uploadfsSettings, { backend: 's3', diff --git a/packages/apostrophe/modules/@apostrophecms/util/index.js b/packages/apostrophe/modules/@apostrophecms/util/index.js index ab47fd0137..38b9e54527 100644 --- a/packages/apostrophe/modules/@apostrophecms/util/index.js +++ b/packages/apostrophe/modules/@apostrophecms/util/index.js @@ -724,7 +724,7 @@ module.exports = { }, // Given a widget or doc, return the appropriate manager module. If the manager // cannot be determined for any reason, undefined is returned. - getManagerOf(object) { + getManagerOf(object, { log = true } = {}) { if (object.metaType === 'doc') { return self.apos.doc.getManager(object.type); } else if (object.metaType === 'widget') { @@ -733,10 +733,10 @@ module.exports = { return self.apos.schema.getArrayManager(object.scopedArrayName); } else if (object.metaType === 'object') { return self.apos.schema.getObjectManager(object.scopedObjectName); - } else { + } else if (log) { self.apos.util.error(`Unsupported metaType in getManagerOf: ${object.metaType}`); - return undefined; } + return undefined; }, // fetch the value at the given path from the object or // array `o`. `path` supports dot notation like MongoDB, and diff --git a/packages/apostrophe/package.json b/packages/apostrophe/package.json index 2d4b651d5d..94c83a7382 100644 --- a/packages/apostrophe/package.json +++ b/packages/apostrophe/package.json @@ -5,10 +5,9 @@ "main": "index.js", "scripts": { "pretest": "npm run lint", - "test": "npm run test:base && npm run test:missing && npm run test:assets && npm run test:esm", - "test:base": "nyc mocha -t 10000 --ignore=test/assets.js", + "test": "npm run test:base && npm run test:missing && npm run test:esm", + "test:base": "nyc mocha -t 10000", "test:missing": "nyc mocha -t 10000 test/add-missing-schema-fields-project/test.js", - "test:assets": "nyc mocha -t 10000 test/assets.js", "test:esm": "mocha -t 1000 test/esm-project/esm.js", "eslint": "eslint .", "eslint-fix": "npm run eslint -- --fix", @@ -38,8 +37,11 @@ "author": "Apostrophe Technologies, Inc.", "license": "MIT", "dependencies": { - "@apostrophecms/emulate-mongo-3-driver": "workspace:^", + "@apostrophecms/db-connect": "workspace:^", "@apostrophecms/vue-material-design-icons": "^1.0.0", + "@babel/core": "^7.29.0", + "@babel/plugin-transform-modules-commonjs": "^7.28.0", + "@babel/plugin-transform-react-jsx": "^7.28.0", "@ctrl/tinycolor": "^4.1.0", "@floating-ui/dom": "^1.5.3", "@opentelemetry/api": "^1.9.0", @@ -127,6 +129,7 @@ "sluggo": "^1.0.0", "sortablejs": "^1.15.0", "sortablejs-vue3": "^1.2.11", + "source-map-support": "^0.5.21", "tiny-emitter": "^2.1.0", "tough-cookie": "^4.0.0", "underscore.string": "^3.3.4", @@ -141,6 +144,7 @@ "xregexp": "^2.0.0" }, "devDependencies": { + "chai": "^4.3.10", "eslint": "^9.39.1", "eslint-config-apostrophe": "workspace:^", "form-data": "^4.0.4", diff --git a/packages/apostrophe/scripts/find-heavy-npm-modules b/packages/apostrophe/scripts/find-heavy-npm-modules old mode 100755 new mode 100644 diff --git a/packages/apostrophe/test-lib/util.js b/packages/apostrophe/test-lib/util.js index 2f60d88f8f..373d8bfd9b 100644 --- a/packages/apostrophe/test-lib/util.js +++ b/packages/apostrophe/test-lib/util.js @@ -1,5 +1,27 @@ const { createId } = require('@paralleldrive/cuid2'); -const mongodbConnect = require('../lib/mongodb-connect'); + +const testDbProtocol = process.env.APOS_TEST_DB_PROTOCOL || 'mongodb'; + +// Build a test database URI for postgres based on the shortName. +// Returns undefined for mongodb, letting the default logic handle it. +function getTestDbUri(shortName) { + if (testDbProtocol === 'postgres') { + // PostgreSQL database names cannot contain hyphens + const dbName = shortName.replace(/-/g, '_'); + return `postgres://localhost:5432/${dbName}`; + } + if (testDbProtocol === 'multipostgres') { + // Multi-schema mode: shared real database, per-test schema + const schemaName = shortName.replace(/-/g, '_').replace(/[^a-zA-Z0-9_]/g, ''); + return `multipostgres://localhost:5432/apos_test-${schemaName}`; + } + if (testDbProtocol === 'sqlite') { + const os = require('os'); + const path = require('path'); + const dbName = shortName.replace(/-/g, '_').replace(/[^a-zA-Z0-9_]/g, ''); + return `sqlite://${path.join(os.tmpdir(), `apos_test_${dbName}.db`)}`; + } +} // Properly clean up an apostrophe instance and drop its // database collections to create a sane environment for the next test. @@ -10,23 +32,23 @@ const mongodbConnect = require('../lib/mongodb-connect'); // If `apos` is null, no work is done. async function destroy(apos) { - if (!apos) { + if (!apos || apos._destroyed) { return; } + apos._destroyed = true; + const dbModule = apos.modules['@apostrophecms/db']; + const { uri } = dbModule; + const dbName = apos.db && (apos.db.databaseName || apos.db._name); await apos.destroy(); - const { uri } = apos.modules['@apostrophecms/db']; - const dbName = apos.db && apos.db.databaseName; - // TODO at some point accommodate nonsense like testing remote databases - // that won't let us use dropDatabase, no shell available etc., but the - // important principle here is that we should not have to have an apos - // object to clean up the database, otherwise we have to get hold of one - // when initialization failed and that's really not apostrophe's concern - if (dbName && uri) { - const client = await mongodbConnect(`${uri}${dbName}`); - const db = client.db(dbName); - await db.dropDatabase(); - await client.close(); + if (!uri || !dbName) { + return; } + // Make a fresh connection (the original was closed by destroy) + // and use it to drop the test database + const client = await dbModule.connectToAdapter(uri); + const db = client.db(dbName); + await db.dropDatabase(); + await client.close(); }; async function create(options = {}) { @@ -55,6 +77,18 @@ async function create(options = {}) { express.options.session.secret = express.options.session.secret || 'test'; config.modules['@apostrophecms/express'] = express; } + // When APOS_TEST_DB_PROTOCOL=postgres, automatically configure the db + // module to use a postgres URI unless already explicitly configured + const testUri = getTestDbUri(config.shortName); + if (testUri) { + config.modules = config.modules || {}; + const dbModule = config.modules['@apostrophecms/db'] || {}; + dbModule.options = dbModule.options || {}; + if (!dbModule.options.uri && !dbModule.options.client) { + dbModule.options.uri = testUri; + } + config.modules['@apostrophecms/db'] = dbModule; + } return require('../index.js')(config); } @@ -151,5 +185,7 @@ module.exports = { loginAs, logout, getUserJar, + getTestDbUri, + testDbProtocol, timeout: (process.env.TEST_TIMEOUT && parseInt(process.env.TEST_TIMEOUT)) || 20000 }; diff --git a/packages/apostrophe/test/add-missing-schema-fields-project/test.js b/packages/apostrophe/test/add-missing-schema-fields-project/test.js index f2c142c3be..9d84f01341 100644 --- a/packages/apostrophe/test/add-missing-schema-fields-project/test.js +++ b/packages/apostrophe/test/add-missing-schema-fields-project/test.js @@ -11,10 +11,23 @@ describe('Apostrophe - add-missing-schema-fields task', function() { let apos; + // When APOS_TEST_DB_PROTOCOL is set, child processes that run `node app.js` + // need the matching APOS_DB_URI so they use the same database as t.create() + const projectCwd = path.resolve(process.cwd(), 'test/add-missing-schema-fields-project/'); + const testDbUri = t.getTestDbUri('add-missing-schema-fields-project'); + const execEnv = testDbUri + ? { + env: { + ...process.env, + APOS_DB_URI: testDbUri + } + } + : {}; + before(async function() { await util.promisify(exec)( 'npm install', - { cwd: path.resolve(process.cwd(), 'test/add-missing-schema-fields-project/') } + { cwd: projectCwd } ); }); @@ -25,7 +38,10 @@ describe('Apostrophe - add-missing-schema-fields task', function() { it('should not run migrations when running the task', async function() { await util.promisify(exec)( 'node app.js @apostrophecms/migration:add-missing-schema-fields', - { cwd: path.resolve(process.cwd(), 'test/add-missing-schema-fields-project/') } + { + cwd: projectCwd, + ...execEnv + } ); apos = await t.create({ @@ -64,7 +80,10 @@ describe('Apostrophe - add-missing-schema-fields task', function() { it('should run migrations when running @apostrophecms/migration:migrate task', async function() { await util.promisify(exec)( 'node app.js @apostrophecms/migration:migrate', - { cwd: path.resolve(process.cwd(), 'test/add-missing-schema-fields-project/') } + { + cwd: projectCwd, + ...execEnv + } ); apos = await t.create({ diff --git a/packages/apostrophe/test/assets.js b/packages/apostrophe/test/assets.js index f5b31454b2..afe6538656 100644 --- a/packages/apostrophe/test/assets.js +++ b/packages/apostrophe/test/assets.js @@ -98,15 +98,75 @@ describe('Assets', function() { retryAssertTrue } = loadUtils(); + // Wait for the chokidar watcher to be ready before writing files. + // Without this, writes can happen before chokidar has finished its + // initial scan, so the change event is never emitted. This is + // especially visible with slower adapters like sqlite. + function waitForWatcherReady(watcher, timeoutMs = 10000) { + if (watcher._readyEmitted) { + return Promise.resolve(); + } + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('Watcher ready timeout')), timeoutMs); + watcher.on('ready', () => { + clearTimeout(timer); + resolve(); + }); + }); + } + + // Many asset tests modify source files in test/modules/ to trigger + // rebuilds, then restore them at the end. If a test fails mid-execution + // the files stay dirty and poison subsequent runs. To prevent this we + // snapshot every mutable file before the suite and restore them + // automatically after each test via afterEach. The before hook also + // cleans up build artifacts and webpack cache from prior runs. + const mutableFiles = [ + 'test/modules/bundle-page/ui/src/extra.js', + 'test/modules/default-page/ui/src/index.js', + 'test/modules/default-page/ui/src/index.scss', + 'test/modules/default-page/ui/public/index.js', + 'test/modules/default-page/ui/public/index.css', + 'test/modules/default-page/ui/apos/components/FakeComponent.vue', + 'test/package-lock.json' + ].map((rel) => path.join(process.cwd(), rel)); + const snapshots = new Map(); + + before(async function() { + // Snapshot every mutable file so afterEach can restore them + for (const file of mutableFiles) { + try { + snapshots.set(file, await fs.readFile(file)); + } catch (e) { + // File might not exist yet, that's OK + } + } + // Start clean: remove build artifacts and cache from prior runs + await deleteBuiltFolders(publicFolderPath, true); + await removeCache(); + }); + after(async function() { await deleteBuiltFolders(publicFolderPath, true); await removeCache(); await t.destroy(apos); }); - afterEach(function() { + afterEach(async function() { // Prevent hang forever if particular tests fail while testing prod. process.env.NODE_ENV = 'development'; + // Restore any files that were modified by the test + for (const [ file, content ] of snapshots) { + try { + const current = await fs.readFile(file); + if (!current.equals(content)) { + await fs.writeFile(file, content); + } + } catch (e) { + // If the file was deleted, restore it + await fs.writeFile(file, content); + } + } }); this.timeout(5 * 60 * 1000); @@ -162,6 +222,7 @@ describe('Assets', function() { }); it('should get webpack extensions from modules and fill extra bundles', async function () { + await t.destroy(apos); const expectedEntryPointsNames = { js: [ 'company', 'main', 'another', 'extra', 'extra2' ], css: [ 'company', 'main', 'extra' ] @@ -309,6 +370,7 @@ describe('Assets', function() { }); it('should build with cache and gain performance', async function() { + await t.destroy(apos); await removeCache(); await removeCache(cacheFolderPath.replace('/webpack-cache', '/changed')); @@ -346,9 +408,11 @@ describe('Assets', function() { assert(meta2['default:apos']); assert(meta2['default:src']); - // Expect at least 40% gain, in reallity it should be 50+ + // Caching should provide a measurable speedup. The threshold is kept + // low (10%) to avoid flaky failures on loaded CI runners where the + // cold run can be fast due to OS-level caching. const gain = (execTime - execTimeCached) / execTime * 100; - assert(gain >= 20, `Expected gain >=20%, got ${gain}%`); + assert(gain >= 10, `Expected gain >=10%, got ${gain}%`); // Modification times assert(meta['default:apos'].mdate); @@ -509,11 +573,11 @@ describe('Assets', function() { assert(apos.asset.restartId); assert(!result.builds); assert(!result.changes); + await waitForWatcherReady(apos.asset.buildWatcher); // Modify asset and rebuild const assetPath = path.join(process.cwd(), 'test/modules/bundle-page/ui/src/extra.js'); const assetPathPublic = path.join(process.cwd(), 'test/public/apos-frontend/default/extra-module-bundle.js'); - const assetContent = fs.readFileSync(assetPath, 'utf-8'); fs.writeFileSync( assetPath, 'export default () => { \'bundle-page-watcher-test-src\'; };\n', @@ -524,34 +588,33 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathPublic, 'utf8')).match(/bundle-page-watcher-test-src/), 'Unable to verify public asset was rebuilt by the watcher', 500, - 10000 + 20000 ); await retryAssertTrue( () => apos.asset.restartId !== restartId, 'Unable to verify restartId has been changed', 500, - 10000 + 20000 ); await retryAssertTrue( () => result.builds.length === 1 && result.builds.includes('src'), 'Unable to verify build "src" has been triggered', 50, - 1000 + 2000 ); await retryAssertTrue( () => result.changes.length === 1 && result.changes[0].includes('modules/bundle-page/ui/src/extra.js'), 'Unable to verify changes contain the proper file', 50, - 1000 + 2000 ); await t.destroy(apos); assert.equal(apos.asset.buildWatcher, null); apos = null; - fs.writeFileSync(assetPath, assetContent, 'utf8'); }); it('should watch and rebuild assets and reload page in development (src)', async function() { @@ -567,12 +630,6 @@ describe('Assets', function() { const assetPathPublicCss = path.join(rootPath, 'test/public/apos-frontend/default/public-bundle.css'); const assetPathAposJs = path.join(rootPath, 'test/public/apos-frontend/default/apos-module-bundle.js'); const assetPathAposCss = path.join(rootPath, 'test/public/apos-frontend/default/apos-bundle.css'); - const assetContentJs = fs.readFileSync(assetPathJs, 'utf-8'); - const assetContentScss = fs.readFileSync(assetPathScss, 'utf-8'); - // Resurrect the default assets content if test has failed - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); - fs.writeFileSync(assetPathScss, assetContentScss, 'utf8'); - apos = await t.create({ root: module, autoBuild: true, @@ -595,6 +652,7 @@ describe('Assets', function() { assert(apos.asset.restartId); assert(!result.builds); assert(!result.changes); + await waitForWatcherReady(apos.asset.buildWatcher); // * modify assets and rebuild fs.writeFileSync( @@ -613,13 +671,13 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathPublicJs, 'utf8')).match(/default-page-watcher-test-src/), 'Unable to verify public JS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); await retryAssertTrue( async () => (await fs.readFile(assetPathPublicCss, 'utf8')).match(/\.default-page-watcher-test-src/), 'Unable to verify public CSS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); // * change is in the apos bundle @@ -627,13 +685,13 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathAposJs, 'utf8')).match(/default-page-watcher-test-src/), 'Unable to verify apos JS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); await retryAssertTrue( async () => (await fs.readFile(assetPathAposCss, 'utf8')).match(/\.default-page-watcher-test-src/), 'Unable to verify apos CSS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); // * page has been restarted @@ -641,7 +699,7 @@ describe('Assets', function() { () => apos.asset.restartId !== restartId, 'Unable to verify restartId has been changed', 500, - 10000 + 20000 ); // * only src related builds were triggered @@ -650,7 +708,7 @@ describe('Assets', function() { result.builds.includes('src'), 'Unable to verify build "src" has been triggered', 50, - 1000 + 2000 ); // * changes detected @@ -665,14 +723,12 @@ describe('Assets', function() { .length === 2, 'Unable to verify changes contain the proper source files', 50, - 1000 + 2000 ); await t.destroy(apos); assert.equal(apos.asset.buildWatcher, null); apos = null; - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); - fs.writeFileSync(assetPathScss, assetContentScss, 'utf8'); }); it('should watch and rebuild assets and reload page in development (public)', async function() { @@ -688,12 +744,6 @@ describe('Assets', function() { const assetPathPublicCss = path.join(rootPath, 'test/public/apos-frontend/default/public-bundle.css'); const assetPathAposJs = path.join(rootPath, 'test/public/apos-frontend/default/apos-module-bundle.js'); const assetPathAposCss = path.join(rootPath, 'test/public/apos-frontend/default/apos-bundle.css'); - const assetContentJs = fs.readFileSync(assetPathJs, 'utf-8'); - const assetContentScss = fs.readFileSync(assetPathCss, 'utf-8'); - // Resurrect the default assets content if test has failed - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); - fs.writeFileSync(assetPathCss, assetContentScss, 'utf8'); - apos = await t.create({ root: module, autoBuild: true, @@ -716,6 +766,7 @@ describe('Assets', function() { assert(apos.asset.restartId); assert(!result.builds); assert(!result.changes); + await waitForWatcherReady(apos.asset.buildWatcher); // * modify assets and rebuild fs.writeFileSync( @@ -734,13 +785,13 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathPublicJs, 'utf8')).match(/default-page-watcher-test-public/), 'Unable to verify public JS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); await retryAssertTrue( async () => (await fs.readFile(assetPathPublicCss, 'utf8')).match(/\.default-page-watcher-test-public/), 'Unable to verify public CSS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); // * change is in the apos bundle @@ -748,13 +799,13 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathAposJs, 'utf8')).match(/default-page-watcher-test-public/), 'Unable to verify apos JS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); await retryAssertTrue( async () => (await fs.readFile(assetPathAposCss, 'utf8')).match(/\.default-page-watcher-test-public/), 'Unable to verify apos CSS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); // * page has been restarted @@ -762,7 +813,7 @@ describe('Assets', function() { () => apos.asset.restartId !== restartId, 'Unable to verify restartId has been changed', 500, - 10000 + 20000 ); // * only public build was triggered @@ -771,7 +822,7 @@ describe('Assets', function() { result.builds.includes('public'), 'Unable to verify build "public" has been triggered', 50, - 1000 + 2000 ); // * changes detected @@ -786,14 +837,12 @@ describe('Assets', function() { .length === 2, 'Unable to verify changes contain the proper source files', 50, - 1000 + 2000 ); await t.destroy(apos); assert.equal(apos.asset.buildWatcher, null); apos = null; - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); - fs.writeFileSync(assetPathCss, assetContentScss, 'utf8'); }); it('should watch and rebuild assets and reload page in development (apos)', async function() { @@ -836,6 +885,7 @@ describe('Assets', function() { assert(apos.asset.restartId); assert(!result.builds); assert(!result.changes); + await waitForWatcherReady(apos.asset.buildWatcher); // * modify assets and rebuild fs.writeFileSync( @@ -850,7 +900,7 @@ describe('Assets', function() { .includes('default-page-watcher-test-apos'), 'Unable to verify apos JS asset was rebuilt by the watcher', 500, - 20000 + 40000 ); // * page has been restarted @@ -858,7 +908,7 @@ describe('Assets', function() { () => apos.asset.restartId !== restartId, 'Unable to verify restartId has been changed', 500, - 10000 + 20000 ); // * only apos build was triggered @@ -867,7 +917,7 @@ describe('Assets', function() { result.builds.includes('apos'), 'Unable to verify build "apos" has been triggered', 50, - 1000 + 2000 ); // * changes detected @@ -877,13 +927,12 @@ describe('Assets', function() { result.changes[0].includes('modules/default-page/ui/apos/components/FakeComponent.vue'), 'Unable to verify changes contain the proper source files', 50, - 1000 + 2000 ); await t.destroy(apos); assert.equal(apos.asset.buildWatcher, null); apos = null; - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); }); it('should watch and recover after build error in development', async function() { @@ -898,9 +947,6 @@ describe('Assets', function() { const assetPathScss = path.join(rootPath, 'test/modules/default-page/ui/src/index.scss'); const assetPathPublicCss = path.join(rootPath, 'test/public/apos-frontend/default/public-bundle.css'); const assetPathAposCss = path.join(rootPath, 'test/public/apos-frontend/default/apos-bundle.css'); - const assetContentScss = '.default-page {color:red;}\n'; - // Resurrect the default assets content if test has failed - fs.writeFileSync(assetPathScss, assetContentScss, 'utf8'); apos = await t.create({ root: module, @@ -924,6 +970,7 @@ describe('Assets', function() { assert(apos.asset.restartId); assert(!result.builds); assert(!result.changes); + await waitForWatcherReady(apos.asset.buildWatcher); // * modify assets and rebuild fs.writeFileSync( @@ -937,7 +984,7 @@ describe('Assets', function() { () => called === 1 && result.builds.length === 0, 'Unable to verify build with error was triggered', 100, - 10000 + 20000 ); // * page has NOT been restarted @@ -945,7 +992,7 @@ describe('Assets', function() { () => apos.asset.restartId === restartId, 'Unable to verify restartId has been changed', 100, - 10000 + 20000 ); // * modify assets and recover @@ -960,7 +1007,7 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathPublicCss, 'utf8')).match(/\.default-page-watcher-test-recover/), 'Unable to verify public CSS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); // * change is in the apos bundle @@ -968,7 +1015,7 @@ describe('Assets', function() { async () => (await fs.readFile(assetPathAposCss, 'utf8')).match(/\.default-page-watcher-test-recover/), 'Unable to verify apos CSS asset was rebuilt by the watcher', 500, - 10000 + 20000 ); // * page has been restarted @@ -976,7 +1023,7 @@ describe('Assets', function() { () => apos.asset.restartId !== restartId, 'Unable to verify restartId has been changed', 500, - 10000 + 20000 ); // * only src related builds were triggered @@ -985,7 +1032,7 @@ describe('Assets', function() { result.builds.includes('src'), 'Unable to verify build "src" have been triggered', 50, - 1000 + 2000 ); // * changes detected @@ -999,13 +1046,12 @@ describe('Assets', function() { .length === 1, 'Unable to verify changes contain the proper source files', 50, - 1000 + 2000 ); await t.destroy(apos); assert.equal(apos.asset.buildWatcher, null); apos = null; - fs.writeFileSync(assetPathScss, assetContentScss, 'utf8'); }); it('should watch but not rebuild assets and not reload page when changes are not in use', async function() { @@ -1023,12 +1069,6 @@ describe('Assets', function() { const assetPathPublicCss = path.join(rootPath, 'test/public/apos-frontend/default/public-bundle.css'); const assetPathAposJs = path.join(rootPath, 'test/public/apos-frontend/default/apos-module-bundle.js'); const assetPathAposCss = path.join(rootPath, 'test/public/apos-frontend/default/apos-bundle.css'); - const assetContentJs = fs.readFileSync(assetPathJs, 'utf-8'); - const assetContentScss = fs.readFileSync(assetPathScss, 'utf-8'); - // Resurrect the default assets content if test has failed - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); - fs.writeFileSync(assetPathScss, assetContentScss, 'utf8'); - apos = await t.create({ root: module, autoBuild: true, @@ -1051,6 +1091,7 @@ describe('Assets', function() { assert(!result.builds); assert(!result.changes); assert.equal(rebuilt, false); + await waitForWatcherReady(apos.asset.buildWatcher); // * modify assets fs.writeFileSync( @@ -1139,8 +1180,6 @@ describe('Assets', function() { await t.destroy(apos); assert.equal(apos.asset.buildWatcher, null); apos = null; - fs.writeFileSync(assetPathJs, assetContentJs, 'utf8'); - fs.writeFileSync(assetPathScss, assetContentScss, 'utf8'); }); it('should watch and rebuild assets in a debounced queue', async function() { @@ -1167,10 +1206,10 @@ describe('Assets', function() { } }); assert(apos.asset.buildWatcher); + await waitForWatcherReady(apos.asset.buildWatcher); const assetPath = path.join(process.cwd(), 'test/modules/bundle-page/ui/src/extra.js'); const assetPathPublic = path.join(process.cwd(), 'test/public/apos-frontend/default/extra-module-bundle.js'); - const assetContent = fs.readFileSync(assetPath, 'utf-8'); // Modify below the debounce rate for (const i of [ 1, 2, 3 ]) { @@ -1195,7 +1234,9 @@ describe('Assets', function() { 5000 ); - // Modify above the debounce rate, test the queue cap + // Modify well above the debounce rate (default 1000ms) so each + // write triggers its own rebuild. Use 2000ms to avoid flaky + // failures on loaded CI runners. timesRebuilt = 0; for (const i of [ 1, 2, 3 ]) { await fs.writeFile( @@ -1203,7 +1244,7 @@ describe('Assets', function() { `export default () => { 'bundle-page-watcher-test-${i}0'; };\n`, 'utf8' ); - await Promise.delay(1050); + await Promise.delay(2000); } await retryAssertTrue( async () => (await fs.readFile(assetPathPublic, 'utf8')).match(/bundle-page-watcher-test-30/), @@ -1220,10 +1261,10 @@ describe('Assets', function() { await t.destroy(apos); apos = null; - fs.writeFileSync(assetPath, assetContent, 'utf8'); }); it('should be able to setup the debounce time', async function() { + await t.destroy(apos); apos = await t.create({ root: module, @@ -1317,6 +1358,7 @@ describe('Assets', function() { }); it('should pass the right options to webpack extensions from all modules', async function() { + await t.destroy(apos); const { extConfig1, extConfig2 } = getWebpackConfigsForExtensionOptions(); apos = await t.create({ @@ -1347,6 +1389,7 @@ describe('Assets', function() { }); it('should allow two modules extending each others to pass options to the same webpack extension', async function() { + await t.destroy(apos); const { extConfig1, extConfig2 } = getWebpackConfigsForExtensionOptions(); apos = await t.create({ diff --git a/packages/apostrophe/test/db-tools.js b/packages/apostrophe/test/db-tools.js new file mode 100644 index 0000000000..2bc33de36a --- /dev/null +++ b/packages/apostrophe/test/db-tools.js @@ -0,0 +1,365 @@ +const assert = require('assert'); +const { execFile } = require('child_process'); +const path = require('path'); +const fs = require('fs'); +const os = require('os'); +const dbConnect = require('@apostrophecms/db-connect'); + +const dumpBin = require.resolve('@apostrophecms/db-connect/bin/apos-db-dump.js'); +const restoreBin = require.resolve('@apostrophecms/db-connect/bin/apos-db-restore.js'); + +const testDbProtocol = process.env.APOS_TEST_DB_PROTOCOL || 'mongodb'; + +function testUri(dbName) { + const dbSafe = dbName.replace(/-/g, '_').replace(/[^a-zA-Z0-9_]/g, ''); + if (testDbProtocol === 'sqlite') { + return `sqlite://${path.join(os.tmpdir(), `${dbSafe}.db`)}`; + } + if (testDbProtocol === 'postgres') { + return `postgres://localhost:5432/${dbSafe}`; + } + const baseUri = process.env.DB_URI || 'mongodb://localhost:27017'; + return `${baseUri}/${dbName}`; +} + +function run(bin, args) { + return new Promise((resolve, reject) => { + execFile(process.execPath, [ bin, ...args ], { + timeout: 30000, + maxBuffer: 50 * 1024 * 1024 + }, (err, stdout, stderr) => { + if (err) { + err.stdout = stdout; + err.stderr = stderr; + return reject(err); + } + resolve({ + stdout, + stderr + }); + }); + }); +} + +async function dropAll(uri) { + let client; + try { + client = await dbConnect(uri); + } catch (e) { + return; + } + const db = client.db(); + const collections = await db.listCollections().toArray(); + for (const col of collections) { + await db.collection(col.name).drop(); + } + await client.close(); +} + +describe('apos-db-dump and apos-db-restore', function () { + this.timeout(30000); + + const sourceUri = testUri('dbtest_dump_source'); + const targetUri = testUri('dbtest_dump_target'); + let tmpFile; + + before(async function () { + tmpFile = path.join(os.tmpdir(), `apos-db-test-${process.pid}.ndjson`); + await dropAll(sourceUri); + await dropAll(targetUri); + }); + + after(async function () { + await dropAll(sourceUri); + await dropAll(targetUri); + try { + fs.unlinkSync(tmpFile); + } catch (e) { + // ignore + } + }); + + it('should dump an empty database without error', async function () { + const { stdout } = await run(dumpBin, [ sourceUri ]); + assert.strictEqual(stdout.trim(), ''); + }); + + it('should dump and restore documents', async function () { + // Insert test data + const client = await dbConnect(sourceUri); + const db = client.db(); + await db.collection('aposDocs').insertMany([ + { + _id: 'doc1', + title: 'Hello', + tags: [ 'a', 'b' ] + }, + { + _id: 'doc2', + title: 'World' + } + ]); + await db.collection('aposCache').insertMany([ + { + _id: 'cache1', + value: 42 + } + ]); + await client.close(); + + // Dump to file + await run(dumpBin, [ sourceUri, `--output=${tmpFile}` ]); + const content = fs.readFileSync(tmpFile, 'utf8'); + const lines = content.split('\n').filter(l => l.trim()); + + // Should have header + docs for each collection + assert(lines.length >= 4, `Expected at least 4 lines, got ${lines.length}`); + + // Every line should be valid JSON + for (const line of lines) { + JSON.parse(line); + } + + // Should have collection headers + const headers = lines + .map(l => JSON.parse(l)) + .filter(e => e._collection && !e._doc); + const collNames = headers.map(h => h._collection).sort(); + assert(collNames.includes('aposDocs')); + assert(collNames.includes('aposCache')); + + // Restore to target + await run(restoreBin, [ targetUri, `--input=${tmpFile}` ]); + + // Verify target has the data + const client2 = await dbConnect(targetUri); + const db2 = client2.db(); + const docs = await db2.collection('aposDocs').find({}).sort({ _id: 1 }).toArray(); + assert.strictEqual(docs.length, 2); + assert.strictEqual(docs[0]._id, 'doc1'); + assert.strictEqual(docs[0].title, 'Hello'); + assert.deepStrictEqual(docs[0].tags, [ 'a', 'b' ]); + assert.strictEqual(docs[1]._id, 'doc2'); + + const cacheDoc = await db2.collection('aposCache').findOne({ _id: 'cache1' }); + assert(cacheDoc); + assert.strictEqual(cacheDoc.value, 42); + await client2.close(); + }); + + it('should preserve Date objects via $date serialization', async function () { + await dropAll(sourceUri); + const client = await dbConnect(sourceUri); + const db = client.db(); + const testDate = new Date('2024-06-15T10:30:00.000Z'); + await db.collection('aposDocs').insertOne({ + _id: 'dateDoc', + createdAt: testDate, + nested: { updatedAt: testDate } + }); + await client.close(); + + // Dump and check format + const { stdout } = await run(dumpBin, [ sourceUri ]); + assert(stdout.includes('"$date"')); + assert(stdout.includes('2024-06-15T10:30:00.000Z'), 'Should contain ISO date string'); + + // Restore and verify dates come back as Date objects + await dropAll(targetUri); + await run(dumpBin, [ sourceUri, `--output=${tmpFile}` ]); + await run(restoreBin, [ targetUri, `--input=${tmpFile}` ]); + + const client2 = await dbConnect(targetUri); + const db2 = client2.db(); + const doc = await db2.collection('aposDocs').findOne({ _id: 'dateDoc' }); + assert(doc.createdAt instanceof Date); + assert.strictEqual(doc.createdAt.toISOString(), '2024-06-15T10:30:00.000Z'); + assert(doc.nested.updatedAt instanceof Date); + assert.strictEqual(doc.nested.updatedAt.toISOString(), '2024-06-15T10:30:00.000Z'); + await client2.close(); + }); + + it('should dump and restore indexes', async function () { + await dropAll(sourceUri); + const client = await dbConnect(sourceUri); + const db = client.db(); + const col = db.collection('aposDocs'); + await col.insertMany([ + { + _id: 'idx1', + slug: 'hello', + price: 10 + }, + { + _id: 'idx2', + slug: 'world', + price: 20 + } + ]); + await col.createIndex({ slug: 1 }); + await col.createIndex({ slug: 1 }, { + unique: true, + name: 'slug_unique' + }); + await col.createIndex({ price: 1 }, { type: 'number' }); + await client.close(); + + // Dump + await run(dumpBin, [ sourceUri, `--output=${tmpFile}` ]); + + const content = fs.readFileSync(tmpFile, 'utf8'); + const header = JSON.parse(content.split('\n')[0]); + assert(header._indexes, 'Header should contain _indexes'); + assert(header._indexes.length >= 2, 'Should have at least 2 custom indexes'); + + // Restore + await dropAll(targetUri); + await run(restoreBin, [ targetUri, `--input=${tmpFile}` ]); + + // Verify indexes exist on target + const client2 = await dbConnect(targetUri); + const db2 = client2.db(); + const indexes = await db2.collection('aposDocs').indexes(); + assert(indexes.find(i => i.key && i.key.slug === 1 && !i.unique), + 'Should have regular slug index'); + assert(indexes.find(i => i.key && i.key.slug === 1 && i.unique), + 'Should have unique slug index'); + + // Verify unique constraint is enforced + try { + await db2.collection('aposDocs').insertOne({ + _id: 'idx3', + slug: 'hello' + }); + assert.fail('Should have rejected duplicate slug'); + } catch (e) { + assert(e.code === 11000 || /duplicate|unique|already exists/i.test(e.message)); + } + + await client2.close(); + }); + + it('should handle piped stdout-to-stdin', async function () { + await dropAll(sourceUri); + const client = await dbConnect(sourceUri); + const db = client.db(); + await db.collection('aposDocs').insertMany([ + { + _id: 'pipe1', + title: 'Piped' + } + ]); + await client.close(); + + // Dump to file, then restore from file (simulating pipe) + const { stdout } = await run(dumpBin, [ sourceUri ]); + + // Write stdout to tmp, restore from it + fs.writeFileSync(tmpFile, stdout); + await dropAll(targetUri); + await run(restoreBin, [ targetUri, `--input=${tmpFile}` ]); + + const client2 = await dbConnect(targetUri); + const db2 = client2.db(); + const doc = await db2.collection('aposDocs').findOne({ _id: 'pipe1' }); + assert(doc); + assert.strictEqual(doc.title, 'Piped'); + await client2.close(); + }); + + it('should handle large collections in batches', async function () { + await dropAll(sourceUri); + const client = await dbConnect(sourceUri); + const db = client.db(); + const docs = []; + for (let i = 0; i < 350; i++) { + docs.push({ + _id: `batch${String(i).padStart(4, '0')}`, + value: i + }); + } + await db.collection('aposDocs').insertMany(docs); + await client.close(); + + // Dump + await run(dumpBin, [ sourceUri, `--output=${tmpFile}` ]); + + const content = fs.readFileSync(tmpFile, 'utf8'); + const lines = content.split('\n').filter(l => l.trim()); + // 1 header + 350 doc lines + assert.strictEqual(lines.length, 351); + + // Docs should be sorted by _id + const docLines = lines.slice(1).map(l => JSON.parse(l)); + for (let i = 1; i < docLines.length; i++) { + assert(docLines[i]._doc._id > docLines[i - 1]._doc._id, + 'Docs should be sorted by _id'); + } + + // Restore and verify count + await dropAll(targetUri); + await run(restoreBin, [ targetUri, `--input=${tmpFile}` ]); + + const client2 = await dbConnect(targetUri); + const db2 = client2.db(); + const count = await db2.collection('aposDocs').countDocuments({}); + assert.strictEqual(count, 350); + await client2.close(); + }); + + it('should restore to a clean state (drop existing data)', async function () { + // Put some pre-existing data in target + const client = await dbConnect(targetUri); + const db = client.db(); + try { + await db.collection('aposDocs').drop(); + } catch (e) { + // ignore + } + await db.collection('aposDocs').insertOne({ + _id: 'old', + title: 'Should be removed' + }); + await client.close(); + + // Set up source with different data + await dropAll(sourceUri); + const client2 = await dbConnect(sourceUri); + const db2 = client2.db(); + await db2.collection('aposDocs').insertOne({ + _id: 'new', + title: 'Fresh data' + }); + await client2.close(); + + // Dump source and restore to target + await run(dumpBin, [ sourceUri, `--output=${tmpFile}` ]); + await run(restoreBin, [ targetUri, `--input=${tmpFile}` ]); + + // Target should only have the new data + const client3 = await dbConnect(targetUri); + const db3 = client3.db(); + const all = await db3.collection('aposDocs').find({}).toArray(); + assert.strictEqual(all.length, 1); + assert.strictEqual(all[0]._id, 'new'); + await client3.close(); + }); + + it('should fail with usage error when no URI is provided', async function () { + try { + await run(dumpBin, []); + assert.fail('Should have exited with error'); + } catch (e) { + assert.strictEqual(e.code, 1); + assert(e.stderr.includes('Usage')); + } + + try { + await run(restoreBin, []); + assert.fail('Should have exited with error'); + } catch (e) { + assert.strictEqual(e.code, 1); + assert(e.stderr.includes('Usage')); + } + }); +}); diff --git a/packages/apostrophe/test/db.js b/packages/apostrophe/test/db.js index 47afa4cf16..7a6134b4de 100644 --- a/packages/apostrophe/test/db.js +++ b/packages/apostrophe/test/db.js @@ -1,6 +1,10 @@ const t = require('../test-lib/test.js'); const assert = require('assert'); +const bogusUri = t.testDbProtocol === 'postgres' + ? 'postgres://this-will-not-work-unless-db-successfully-overrides-it/fail' + : 'mongodb://this-will-not-work-unless-db-successfully-overrides-it/fail'; + describe('Db', function() { let apos, apos2; @@ -25,23 +29,28 @@ describe('Db', function() { assert(doc); }); - it('should be able to launch a second instance reusing the connection', async function() { - // Often takes too long otherwise - this.timeout(10000); - apos2 = await t.create({ - root: module, - modules: { - '@apostrophecms/db': { - options: { - client: apos.dbClient, - uri: 'mongodb://this-will-not-work-unless-db-successfully-overrides-it/fail' + // Client reuse with a different database name is only supported in + // mongodb and multipostgres mode, not simple postgres (which has no + // schema isolation) + if (t.testDbProtocol !== 'postgres') { + it('should be able to launch a second instance reusing the connection', async function() { + // Often takes too long otherwise + this.timeout(10000); + apos2 = await t.create({ + root: module, + modules: { + '@apostrophecms/db': { + options: { + client: apos.dbClient, + uri: bogusUri + } } } - } - }); + }); - const doc = await apos2.doc.db.findOne(); + const doc = await apos2.doc.db.findOne(); - assert(doc); - }); + assert(doc); + }); + } }); diff --git a/packages/apostrophe/test/default-adapter.js b/packages/apostrophe/test/default-adapter.js new file mode 100644 index 0000000000..ad8bc628e8 --- /dev/null +++ b/packages/apostrophe/test/default-adapter.js @@ -0,0 +1,256 @@ +const assert = require('assert'); +const path = require('path'); + +describe('Default Adapter', function() { + + this.timeout(20000); + + // Save and restore env vars + let savedAposDefaultDbAdapter; + let savedAposDbUri; + let savedAposMongdbUri; + + beforeEach(function() { + savedAposDefaultDbAdapter = process.env.APOS_DEFAULT_DB_ADAPTER; + savedAposDbUri = process.env.APOS_DB_URI; + savedAposMongdbUri = process.env.APOS_MONGODB_URI; + delete process.env.APOS_DEFAULT_DB_ADAPTER; + delete process.env.APOS_DB_URI; + delete process.env.APOS_MONGODB_URI; + }); + + afterEach(function() { + if (savedAposDefaultDbAdapter !== undefined) { + process.env.APOS_DEFAULT_DB_ADAPTER = savedAposDefaultDbAdapter; + } else { + delete process.env.APOS_DEFAULT_DB_ADAPTER; + } + if (savedAposDbUri !== undefined) { + process.env.APOS_DB_URI = savedAposDbUri; + } else { + delete process.env.APOS_DB_URI; + } + if (savedAposMongdbUri !== undefined) { + process.env.APOS_MONGODB_URI = savedAposMongdbUri; + } else { + delete process.env.APOS_MONGODB_URI; + } + }); + + // These tests verify URI construction by examining module internals + // without needing actual database connections. + + it('builds mongodb:// URI by default', function() { + const uri = buildUri({ shortName: 'mysite' }); + assert(uri.startsWith('mongodb://')); + assert(uri.includes('localhost:27017')); + assert(uri.includes('/mysite')); + }); + + it('builds mongodb:// URI when defaultAdapter is "mongodb"', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { defaultAdapter: 'mongodb' } + }); + assert(uri.startsWith('mongodb://')); + assert(uri.includes('/mysite')); + }); + + it('builds sqlite:// URI when defaultAdapter is "sqlite"', function() { + const uri = buildUri({ + shortName: 'mysite', + rootDir: '/app', + dbOptions: { defaultAdapter: 'sqlite' } + }); + assert(uri.startsWith('sqlite://')); + assert(uri.includes(path.join('data', 'mysite.sqlite'))); + }); + + it('builds postgres:// URI when defaultAdapter is "postgres"', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { defaultAdapter: 'postgres' } + }); + assert(uri.startsWith('postgres://')); + assert(uri.includes('localhost:5432')); + assert(uri.includes('/mysite')); + }); + + it('builds multipostgres:// URI when defaultAdapter is "multipostgres"', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { defaultAdapter: 'multipostgres' } + }); + assert(uri.startsWith('multipostgres://')); + assert(uri.includes('localhost:5432')); + assert(uri.includes('/mysite')); + }); + + it('includes URI-encoded credentials for postgres', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { + defaultAdapter: 'postgres', + user: 'admin', + password: 'p@ss:word/special' + } + }); + assert(uri.startsWith('postgres://')); + assert(uri.includes('admin')); + assert(uri.includes(encodeURIComponent('p@ss:word/special'))); + assert(!uri.includes('p@ss:word/special')); + }); + + it('includes URI-encoded credentials for mongodb', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { + defaultAdapter: 'mongodb', + user: 'admin', + password: 'p@ss:word' + } + }); + assert(uri.startsWith('mongodb://')); + assert(uri.includes(encodeURIComponent('p@ss:word'))); + }); + + it('APOS_DEFAULT_DB_ADAPTER env var overrides the option', function() { + process.env.APOS_DEFAULT_DB_ADAPTER = 'postgres'; + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { defaultAdapter: 'mongodb' } + }); + assert(uri.startsWith('postgres://')); + }); + + it('throws for invalid adapter name', function() { + assert.throws(() => { + buildUri({ + shortName: 'mysite', + dbOptions: { defaultAdapter: 'invalid' } + }); + }, /Invalid defaultAdapter/); + }); + + it('explicit uri option overrides defaultAdapter', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { + defaultAdapter: 'postgres', + uri: 'mongodb://custom:27017/other' + } + }); + assert.strictEqual(uri, 'mongodb://custom:27017/other'); + }); + + it('APOS_DB_URI env var overrides everything', function() { + process.env.APOS_DB_URI = 'mongodb://envhost:27017/envdb'; + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { defaultAdapter: 'postgres' } + }); + assert.strictEqual(uri, 'mongodb://envhost:27017/envdb'); + }); + + it('honors custom host and port for postgres', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { + defaultAdapter: 'postgres', + host: 'dbserver', + port: 5433 + } + }); + assert(uri.includes('dbserver:5433')); + }); + + it('honors custom host and port for mongodb', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { + defaultAdapter: 'mongodb', + host: 'mongohost', + port: 27018 + } + }); + assert(uri.includes('mongohost:27018')); + }); + + it('uses shortName as database name by default', function() { + const uri = buildUri({ shortName: 'my-app' }); + assert(uri.endsWith('/my-app')); + }); + + it('uses name option over shortName when provided', function() { + const uri = buildUri({ + shortName: 'my-app', + dbOptions: { name: 'custom-db' } + }); + assert(uri.includes('/custom-db')); + }); + + it('ignores host/port/user/password for sqlite', function() { + const uri = buildUri({ + shortName: 'mysite', + dbOptions: { + defaultAdapter: 'sqlite', + host: 'shouldbeignored', + port: 9999, + user: 'nobody', + password: 'nothing' + } + }); + assert(uri.startsWith('sqlite://')); + assert(!uri.includes('shouldbeignored')); + assert(!uri.includes('9999')); + assert(!uri.includes('nobody')); + }); +}); + +// Helper: simulate the URI construction logic from the db module +// without actually connecting. This extracts the same logic path. +function buildUri(options = {}) { + const escapeHost = require('../lib/escape-host.js'); + const shortName = options.shortName || 'test-app'; + const rootDir = options.rootDir || '/tmp/test-app'; + const dbOptions = { ...(options.dbOptions || {}) }; + + // Simulate the connectToDb URI construction + const viaEnv = process.env.APOS_DB_URI || process.env.APOS_MONGODB_URI; + if (viaEnv) { + return viaEnv; + } + if (dbOptions.uri) { + return dbOptions.uri; + } + + const validAdapters = [ 'mongodb', 'sqlite', 'postgres', 'multipostgres' ]; + const adapter = process.env.APOS_DEFAULT_DB_ADAPTER || dbOptions.defaultAdapter || 'mongodb'; + if (!validAdapters.includes(adapter)) { + throw new Error(`Invalid defaultAdapter: "${adapter}". Must be one of: ${validAdapters.join(', ')}`); + } + + if (!dbOptions.name) { + dbOptions.name = shortName; + } + + if (adapter === 'sqlite') { + const path = require('path'); + return `sqlite://${path.resolve(rootDir, 'data', dbOptions.name + '.sqlite')}`; + } + + const credentials = dbOptions.user + ? encodeURIComponent(dbOptions.user) + ':' + encodeURIComponent(dbOptions.password) + '@' + : ''; + + if (adapter === 'mongodb') { + const host = dbOptions.host || 'localhost'; + const port = dbOptions.port || 27017; + return 'mongodb://' + credentials + escapeHost(host) + ':' + port + '/' + dbOptions.name; + } + + // postgres or multipostgres + const host = dbOptions.host || 'localhost'; + const port = dbOptions.port || 5432; + return adapter + '://' + credentials + escapeHost(host) + ':' + port + '/' + dbOptions.name; +} diff --git a/packages/apostrophe/test/external-front.js b/packages/apostrophe/test/external-front.js index a750fce038..e12e5157c1 100644 --- a/packages/apostrophe/test/external-front.js +++ b/packages/apostrophe/test/external-front.js @@ -17,12 +17,430 @@ describe('External Front', function() { it('apostrophe should initialize normally', async function() { apos = await t.create({ - root: module + root: module, + modules: { + product: { + extend: '@apostrophecms/piece-type', + options: { + alias: 'product' + }, + fields: { + add: { + main: { + type: 'area', + options: { + widgets: { + '@apostrophecms/rich-text': {} + } + } + }, + extra: { + type: 'area', + options: { + widgets: { + '@apostrophecms/rich-text': {} + } + } + } + } + } + } + } }); assert(apos.page.__meta.name === '@apostrophecms/page'); }); + // A doc as it would look in memory, with `extra` simulating an area field + // added to the schema after the doc was created (so it has no value). + function productMissingExtra() { + const doc = apos.product.newInstance(); + doc._id = 'product1:en:published'; + doc.metaType = 'doc'; + doc.title = 'Product 1'; + doc.slug = 'product-1'; + delete doc.extra; + return doc; + } + + it('missingSchemaAreas returns unfilled area fields, ignores non-schema objects', function() { + const missing = apos.template.missingSchemaAreas(productMissingExtra()); + assert.strictEqual(missing.length, 1); + assert.strictEqual(missing[0].name, 'extra'); + + // An area object carries `_edit` but has no schema manager: returns [] + // quietly (getManagerOf called with log: false). + const area = { + metaType: 'area', + _edit: true + }; + assert.deepStrictEqual(apos.template.missingSchemaAreas(area), []); + }); + + it('annotateDocForExternalFront materializes missing areas on an editable doc', async function() { + const doc = productMissingExtra(); + doc._edit = true; + // As loaded for an editor, existing areas carry _edit too + doc.main._edit = true; + + await apos.template.annotateDocForExternalFront(doc); + + // Existing area still annotated, unchanged behavior + assert(doc.main.field && doc.main.field.name === 'main'); + assert(doc.main.options); + // Carries the provenance signal, and is never flagged as orphan + assert.strictEqual(doc.main._aposAnnotated, true); + assert.strictEqual(doc.main._isOrphan, undefined); + + // Missing area added as an empty, editable, annotated area + assert(doc.extra, 'extra area was materialized'); + assert.strictEqual(doc.extra.metaType, 'area'); + assert.deepStrictEqual(doc.extra.items, []); + assert.strictEqual(doc.extra._edit, true); + assert.strictEqual(doc.extra._docId, doc._id); + assert(doc.extra._id, 'has an id'); + assert(doc.extra.field && doc.extra.field.name === 'extra'); + assert(doc.extra.options, 'annotated with options'); + assert(Array.isArray(doc.extra.choices)); + assert.strictEqual(doc.extra._aposAnnotated, true); + assert.strictEqual(doc.extra._isOrphan, undefined); + }); + + it('flags a genuine orphan area, leaves valid areas alone, and never persists the flag', async function() { + // Insert a doc with an area whose field is no longer in the schema. + const docId = 'orphan-test:en:draft'; + await apos.doc.db.deleteOne({ _id: docId }); + await apos.doc.db.insertOne({ + _id: docId, + type: 'product', + metaType: 'doc', + aposMode: 'draft', + aposDocId: 'orphan-test', + aposLocale: 'en:draft', + title: 'Orphan Test', + slug: 'orphan-test', + main: { + metaType: 'area', + _id: 'orphan-main', + items: [] + }, + // `ghost` is not a field in the product schema (simulates a removed field) + ghost: { + metaType: 'area', + _id: 'orphan-ghost', + items: [] + } + }); + + const doc = await apos.doc.db.findOne({ _id: docId }); + doc._edit = true; + doc.main._edit = true; + doc.ghost._edit = true; + + await apos.template.annotateDocForExternalFront(doc); + + // The annotator owns the doc, so the orphan is flagged — and has no field. + // It is NOT marked `_aposAnnotated` (that signals a fully annotated area). + assert.strictEqual(doc.ghost._isOrphan, true, 'orphan flagged'); + assert.strictEqual(doc.ghost.field, undefined, 'orphan has no schema field'); + assert.strictEqual(doc.ghost._aposAnnotated, undefined, 'orphan is not _aposAnnotated'); + // The valid area is annotated normally and never flagged orphan. + assert(doc.main.field && doc.main._isOrphan === undefined); + assert.strictEqual(doc.main._aposAnnotated, true); + + // Neither flag is written to the database. + const persisted = await apos.doc.db.findOne({ _id: docId }); + assert.strictEqual(persisted.ghost._isOrphan, undefined, 'flag not persisted'); + assert.strictEqual(persisted.main._isOrphan, undefined, 'flag not persisted'); + assert.strictEqual(persisted.main._aposAnnotated, undefined, 'signal not persisted'); + + await apos.doc.db.deleteOne({ _id: docId }); + }); + + it('annotateDocForExternalFront leaves missing areas alone on a non-editable doc', async function() { + const doc = productMissingExtra(); + + await apos.template.annotateDocForExternalFront(doc); + + assert.strictEqual(doc.extra, undefined, 'extra not added for anonymous'); + // Existing area annotated as before + assert(doc.main.field && doc.main.field.name === 'main'); + }); + + it('annotateDocForExternalFront persists materialized areas at their schema path with the same _id sent to the UI', async function() { + // Insert a doc directly with only `main`, no `extra`, simulating a field + // added to the schema after the doc was created. + const docId = 'persist-test:en:draft'; + await apos.doc.db.deleteOne({ _id: docId }); + await apos.doc.db.insertOne({ + _id: docId, + type: 'product', + metaType: 'doc', + aposMode: 'draft', + aposDocId: 'persist-test', + aposLocale: 'en:draft', + title: 'Persist Test', + slug: 'persist-test', + main: { + metaType: 'area', + _id: 'main-id-persist', + items: [] + } + }); + + // Load the doc and mark editable (mirroring what doc-type load does) + const doc = await apos.doc.db.findOne({ _id: docId }); + doc._edit = true; + doc.main._edit = true; + + await apos.template.annotateDocForExternalFront(doc); + + // In-memory: extra was materialized and annotated + assert(doc.extra && doc.extra._id, 'extra has an id in memory'); + const inMemoryId = doc.extra._id; + + // In the DB: extra was written at the schema path with the same _id, so a + // subsequent editor patch using @.items will resolve + const persisted = await apos.doc.db.findOne({ _id: docId }); + assert(persisted.extra, 'extra was persisted'); + assert.strictEqual(persisted.extra.metaType, 'area'); + assert.deepStrictEqual(persisted.extra.items, []); + assert.strictEqual( + persisted.extra._id, inMemoryId, + 'persisted _id matches the one sent to the UI' + ); + + // Idempotent: a second annotate keeps the same persisted _id (the + // $eq: null condition prevents overwrite) + const doc2 = await apos.doc.db.findOne({ _id: docId }); + doc2._edit = true; + delete doc2.extra; // simulate the missing-area branch firing again + await apos.template.annotateDocForExternalFront(doc2); + const persistedAgain = await apos.doc.db.findOne({ _id: docId }); + assert.strictEqual( + persistedAgain.extra._id, inMemoryId, + 'persisted _id is unchanged on re-annotate' + ); + + await apos.doc.db.deleteOne({ _id: docId }); + }); + + it('does not write missing areas at an in-memory path when a relationship target needs one (regression)', async function() { + // The editor's in-memory graph contains loaded relationship data. A widget + // in the host doc relates to a SEPARATE editable doc that is missing a + // schema area (added after it was created). The area must be stubbed at the + // related doc's OWN path — never at a path derived from the host's + // in-memory traversal, which would write into the wrong document and pad + // arrays with nulls. This reproduces the production corruption that left + // null items and stray fragments in published docs. + const hostId = 'reg-host:en:draft'; + const relatedId = 'reg-related:en:draft'; + await apos.doc.db.deleteMany({ _id: { $in: [ hostId, relatedId ] } }); + + // Host has both areas filled (nothing missing of its own) and a single + // rich-text widget in `main`. + await apos.doc.db.insertOne({ + _id: hostId, + type: 'product', + metaType: 'doc', + aposMode: 'draft', + aposDocId: 'reg-host', + aposLocale: 'en:draft', + title: 'Host', + slug: 'reg-host', + main: { + metaType: 'area', + _id: 'reg-host-main', + items: [ + { + _id: 'reg-host-widget', + metaType: 'widget', + type: '@apostrophecms/rich-text', + content: '

hi

' + } + ] + }, + extra: { + metaType: 'area', + _id: 'reg-host-extra', + items: [] + } + }); + // Related has `main` but no `extra` (field added to the schema later). + await apos.doc.db.insertOne({ + _id: relatedId, + type: 'product', + metaType: 'doc', + aposMode: 'draft', + aposDocId: 'reg-related', + aposLocale: 'en:draft', + title: 'Related', + slug: 'reg-related', + main: { + metaType: 'area', + _id: 'reg-related-main', + items: [] + } + }); + + const hostStored = await apos.doc.db.findOne({ _id: hostId }); + + // Build the in-memory graph: the host widget carries a loaded relationship + // to the related doc, which is editable and missing `extra`. + const related = await apos.doc.db.findOne({ _id: relatedId }); + related._edit = true; + related._docId = relatedId; + related.main._edit = true; + delete related.extra; + + const host = await apos.doc.db.findOne({ _id: hostId }); + host._edit = true; + host._docId = hostId; + host.main._edit = true; + host.main.items[0]._docId = hostId; + host.main.items[0]._related = [ related ]; + + await apos.template.annotateDocForExternalFront(host); + + // Related doc: `extra` stubbed at its OWN top-level path, clean, and its + // existing `main` is untouched (no host-relative path leaked in). + const relatedAfter = await apos.doc.db.findOne({ _id: relatedId }); + assert(relatedAfter.extra, 'extra stubbed on the related doc'); + assert.strictEqual(relatedAfter.extra.metaType, 'area'); + assert.deepStrictEqual(relatedAfter.extra.items, []); + assert.strictEqual(relatedAfter.extra._id, related.extra._id, 'same _id as in memory'); + assert.strictEqual(relatedAfter.main.items.length, 0, 'related main not padded'); + assert.ok(!Object.prototype.hasOwnProperty.call(relatedAfter, '0'), 'no numeric-key fragment'); + + // Host doc: completely untouched in the database. + const hostAfter = await apos.doc.db.findOne({ _id: hostId }); + assert.strictEqual(hostAfter.main.items.length, 1, 'host main.items not extended'); + assert( + hostAfter.main.items.every(i => i && i.metaType === 'widget'), + 'no null/typeless items in host' + ); + assert.deepStrictEqual(hostAfter, hostStored, 'host doc unchanged in the DB'); + + await apos.doc.db.deleteMany({ _id: { $in: [ hostId, relatedId ] } }); + }); + + it('annotateAreaForExternalFront drops corrupt items so they never reach the front end', function() { + const field = { + name: 'main', + options: { + widgets: { '@apostrophecms/rich-text': {} } + } + }; + const area = { + metaType: 'area', + _id: 'guard-area', + _docId: 'guard-doc:en:published', + items: [ + { + _id: 'w1', + metaType: 'widget', + type: '@apostrophecms/rich-text', + content: '

ok

' + }, + // The two shapes the production corruption produced. A `null` left in + // place would crash the Astro area renderer (`...item._options`). + null, + { + _id: 'frag', + foo: 'bar' + } + ] + }; + + assert.doesNotThrow(() => { + apos.template.annotateAreaForExternalFront(field, area, { scene: 'apos' }); + }); + + // Corrupt items are removed; only the valid, annotated widget remains. + assert.strictEqual(area.items.length, 1, 'corrupt items dropped'); + assert.strictEqual(area.items[0]._id, 'w1'); + assert(area.items[0]._options, 'valid widget annotated'); + assert.strictEqual(area.items[0]._docId, area._docId, 'valid widget got _docId'); + }); + + it('annotateAreaForExternalFront keeps an unknown widget type un-annotated instead of throwing', function() { + const field = { + name: 'main', + options: { + widgets: { '@apostrophecms/rich-text': {} } + } + }; + const area = { + metaType: 'area', + _id: 'unknown-area', + _docId: 'unknown-doc:en:published', + items: [ + { + _id: 'w1', + metaType: 'widget', + type: '@apostrophecms/rich-text', + content: '

ok

' + }, + // A real widget whose module is not registered (e.g. `custom-layout`). + { + _id: 'w2', + metaType: 'widget', + type: 'definitely-not-a-registered-widget' + } + ] + }; + + // No throw — a missing widget module must not 500 the whole render. + assert.doesNotThrow(() => { + apos.template.annotateAreaForExternalFront(field, area, { scene: 'apos' }); + }); + + // The unknown widget is preserved (so its content survives a save) but left + // un-annotated; the front end skips it. + assert.strictEqual(area.items.length, 2, 'unknown-type item preserved'); + assert(area.items[0]._options, 'valid widget annotated'); + assert.strictEqual(area.items[1].type, 'definitely-not-a-registered-widget'); + assert.strictEqual(area.items[1]._options, undefined, 'unknown widget not annotated'); + }); + + it('addMissingArea honors throwIfNotFound (tag behavior) and defaults to graceful (annotator)', async function() { + // Doc-backed parent whose document is not in the database (the missing-doc + // race the `{% area %}` tag historically treated as notfound). + const ghost = { + _id: 'ghost:en:published', + metaType: 'doc', + type: 'product' + }; + + // Opt-in (tag): throws notfound rather than persisting nothing silently. + await assert.rejects( + () => apos.area.addMissingArea(ghost, 'extra', { throwIfNotFound: true }), + err => err && err.name === 'notfound' + ); + // The in-memory stub is still attached for the caller. + assert(ghost.extra && ghost.extra.metaType === 'area'); + + // Default (annotator): degrades to an in-memory stub, never throws. + const ghost2 = { + _id: 'ghost2:en:published', + metaType: 'doc', + type: 'product' + }; + const area = await apos.area.addMissingArea(ghost2, 'extra'); + assert.strictEqual(area.metaType, 'area'); + assert(area._id, 'in-memory stub has an _id'); + assert.deepStrictEqual(area.items, []); + assert.strictEqual(ghost2.extra, area, 'stub attached to the parent'); + + // A parent with no docId is never an error, even with throwIfNotFound. + const unsaved = { + metaType: 'doc', + type: 'product' + }; + const unsavedArea = await apos.area.addMissingArea(unsaved, 'extra', { throwIfNotFound: true }); + assert.strictEqual(unsavedArea.metaType, 'area'); + }); + it('fetch home with external front', async function() { const data = await await apos.http.get('/', { headers: { diff --git a/packages/apostrophe/test/job.js b/packages/apostrophe/test/job.js index 75b1ad47b9..803368256e 100644 --- a/packages/apostrophe/test/job.js +++ b/packages/apostrophe/test/job.js @@ -311,7 +311,7 @@ describe('Job module', function() { req, async function(_req, reporters) { let count = 1; - reporters.setTotal(articleIds.length); + await reporters.setTotal(articleIds.length); for (const id of articleIds) { await delay(3); diff --git a/packages/apostrophe/test/modules/jsx-area-test/index.js b/packages/apostrophe/test/modules/jsx-area-test/index.js new file mode 100644 index 0000000000..e750c729b2 --- /dev/null +++ b/packages/apostrophe/test/modules/jsx-area-test/index.js @@ -0,0 +1,23 @@ +module.exports = { + extend: '@apostrophecms/piece-type', + options: { + alias: 'jsxAreaTest', + name: 'jsx-area-test', + label: 'JSX Area Test' + }, + fields: { + add: { + main: { + type: 'area', + label: 'Main', + options: { + widgets: { + '@apostrophecms/rich-text': {}, + 'jsx-async': {}, + 'jsx-ctx': {} + } + } + } + } + } +}; diff --git a/packages/apostrophe/test/modules/jsx-area-test/views/bad-area.jsx b/packages/apostrophe/test/modules/jsx-area-test/views/bad-area.jsx new file mode 100644 index 0000000000..d9ce539fec --- /dev/null +++ b/packages/apostrophe/test/modules/jsx-area-test/views/bad-area.jsx @@ -0,0 +1,7 @@ +export default function (data, { Area }) { + return ( +
+ +
+ ); +} diff --git a/packages/apostrophe/test/modules/jsx-area-test/views/with-area-ctx.jsx b/packages/apostrophe/test/modules/jsx-area-test/views/with-area-ctx.jsx new file mode 100644 index 0000000000..da3cb9b266 --- /dev/null +++ b/packages/apostrophe/test/modules/jsx-area-test/views/with-area-ctx.jsx @@ -0,0 +1,13 @@ +export default function (data, { Area }) { + return ( +
+ +
+ ); +} diff --git a/packages/apostrophe/test/modules/jsx-area-test/views/with-area.jsx b/packages/apostrophe/test/modules/jsx-area-test/views/with-area.jsx new file mode 100644 index 0000000000..99a1985f53 --- /dev/null +++ b/packages/apostrophe/test/modules/jsx-area-test/views/with-area.jsx @@ -0,0 +1,7 @@ +export default function (data, { Area }) { + return ( +
+ +
+ ); +} diff --git a/packages/apostrophe/test/modules/jsx-area-test/views/with-widget-ctx.jsx b/packages/apostrophe/test/modules/jsx-area-test/views/with-widget-ctx.jsx new file mode 100644 index 0000000000..17aedd9bd5 --- /dev/null +++ b/packages/apostrophe/test/modules/jsx-area-test/views/with-widget-ctx.jsx @@ -0,0 +1,12 @@ +export default function (data, { Widget }) { + return ( +
+ +
+ ); +} diff --git a/packages/apostrophe/test/modules/jsx-area-test/views/with-widget.jsx b/packages/apostrophe/test/modules/jsx-area-test/views/with-widget.jsx new file mode 100644 index 0000000000..a1e801e38b --- /dev/null +++ b/packages/apostrophe/test/modules/jsx-area-test/views/with-widget.jsx @@ -0,0 +1,7 @@ +export default function (data, { Widget }) { + return ( +
+ +
+ ); +} diff --git a/packages/apostrophe/test/modules/jsx-async-widget/index.js b/packages/apostrophe/test/modules/jsx-async-widget/index.js new file mode 100644 index 0000000000..6c4bdf2899 --- /dev/null +++ b/packages/apostrophe/test/modules/jsx-async-widget/index.js @@ -0,0 +1,6 @@ +module.exports = { + extend: '@apostrophecms/widget-type', + options: { + label: 'JSX Async Widget' + } +}; diff --git a/packages/apostrophe/test/modules/jsx-async-widget/views/widget.jsx b/packages/apostrophe/test/modules/jsx-async-widget/views/widget.jsx new file mode 100644 index 0000000000..9ad8ce16e8 --- /dev/null +++ b/packages/apostrophe/test/modules/jsx-async-widget/views/widget.jsx @@ -0,0 +1,11 @@ +export default function (data, { Component }) { + return ( +
+ +
+ ); +} diff --git a/packages/apostrophe/test/modules/jsx-bridge-test/index.js b/packages/apostrophe/test/modules/jsx-bridge-test/index.js new file mode 100644 index 0000000000..f053ebf797 --- /dev/null +++ b/packages/apostrophe/test/modules/jsx-bridge-test/index.js @@ -0,0 +1 @@ +module.exports = {}; diff --git a/packages/apostrophe/test/modules/jsx-bridge-test/views/cross-module.jsx b/packages/apostrophe/test/modules/jsx-bridge-test/views/cross-module.jsx new file mode 100644 index 0000000000..f80d7e44fb --- /dev/null +++ b/packages/apostrophe/test/modules/jsx-bridge-test/views/cross-module.jsx @@ -0,0 +1,7 @@ +export default function (data, { Template }) { + return ( +
+