add addMissingSchemaFields task (#5101)

* add addMissingSchemaFields task

* add skipMigration flag

* add node 24 support

* move migration handler from apostrophe index to migration after handler

* add migration requirements handler

---------

Co-authored-by: Harouna Traoré <haroun@users.noreply.github.com>

Author: haroun

.github/workflows/main.yml

@@ -18,21 +18,21 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [20, 22]
+        node-version: [20, 22, 24]
         mongodb-version: [6.0, 7.0, 8.0]
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       - name: Git checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: ${{ matrix.node-version }}
 
       - name: Start MongoDB
-        uses: supercharge/mongodb-github-action@1.11.0
+        uses: supercharge/mongodb-github-action@1.12.0
         with:
           mongodb-version: ${{ matrix.mongodb-version }}
 

CHANGELOG.md

@@ -4,13 +4,15 @@
 
 ### Adds
 
+* Add `@apostrophecms/migration:add-missing-schema-fields` task. This task does not run database migrations.
 * Translation strings added for the layout- and layout-column-widgets.
 * New `box` schema field type
 * When switching locale from the doc editor, ask if the user wants to localize the current document in the target locale or want to start a blank document.
 * Introduced a new `longPolling: false` option for the `@apostrophecms/notification` module. This eliminates long-pending requests when logged in, but also slows down the delivery of notifications. The behavior can be tuned further via the `pollingInterval` option, which defaults to `5000` milliseconds.
 
 ### Changes
 
+* `@apostrophecms/migration:requirements` handler now runs the migration requirements like `insertIfMissing`, `implementParkAllInDefaultLocale`, `replicate` and `implementParkAllInOtherLocales`.
 * Bump nodemailer to v7.x.
 
 ### Fixes

index.js

@@ -323,21 +323,7 @@ async function apostrophe(options, telemetry, rootSpan) {
     await self.emit('modulesRegistered'); // formerly modulesReady
     self.apos.schema.validateAllSchemas();
     self.apos.schema.registerAllSchemas();
-    await self.apos.lock.withLock('@apostrophecms/migration:migrate', async () => {
-      await self.apos.migration.migrate(self.argv);
-      // Inserts the global doc in the default locale if it does not exist;
-      // same for other singleton piece types registered by other modules
-      for (const apostropheModule of Object.values(self.modules)) {
-        if (self.instanceOf(apostropheModule, '@apostrophecms/piece-type') && apostropheModule.options.singletonAuto) {
-          await apostropheModule.insertIfMissing();
-        }
-      }
-      await self.apos.page.implementParkAllInDefaultLocale();
-      await self.apos.doc.replicate(); // emits beforeReplicate and afterReplicate events
-      // Replicate will have created the parked pages across locales if needed,
-      // but we may still need to reset parked properties
-      await self.apos.page.implementParkAllInOtherLocales();
-    });
+    await self.apos.migration.migrate(self.argv);
     await self.emit('ready'); // formerly afterInit
 
     if (self.taskRan) {

modules/@apostrophecms/migration/index.js

@@ -14,13 +14,53 @@ const addMissingSchemaFields = require('./lib/addMissingSchemaFields.js');
 // is difficult to guarantee, you may wish to write a task instead.
 
 module.exports = {
-  options: { alias: 'migration' },
+  options: {
+    alias: 'migration',
+    skipMigrationTasks: [
+      '@apostrophecms/migration:add-missing-schema-fields'
+    ]
+  },
   async init(self) {
     self.migrations = [];
     await self.enableCollection();
   },
   handlers(self) {
     return {
+      'apostrophe:modulesRegistered': {
+        setSkipMigration() {
+          if (
+            self.apos.isTask() &&
+            self.options.skipMigrationTasks.includes(self.apos.argv._.at(0))
+          ) {
+            self.apos.skipMigration = true;
+          }
+        }
+      },
+      before: {
+        async addMissingSchemaFields() {
+          await self.addMissingSchemaFields();
+        }
+      },
+      requirements: {
+        async insertRequired() {
+          // Inserts the global doc in the default locale if it does not exist;
+          // same for other singleton piece types registered by other modules
+          for (const apostropheModule of Object.values(self.apos.modules)) {
+            if (
+              self.apos.instanceOf(apostropheModule, '@apostrophecms/piece-type') &&
+              apostropheModule.options.singletonAuto
+            ) {
+              await apostropheModule.insertIfMissing();
+            }
+          }
+          await self.apos.page.implementParkAllInDefaultLocale();
+          await self.apos.doc.replicate();
+          // emits beforeReplicate and afterReplicate events
+          // Replicate will have created the parked pages across locales if needed,
+          // but we may still need to reset parked properties
+          await self.apos.page.implementParkAllInOtherLocales();
+        }
+      },
       'apostrophe:ready': {
         addSortifyMigrations() {
           const managers = self.apos.doc.managers;
@@ -41,11 +81,6 @@ module.exports = {
             });
           });
         }
-      },
-      before: {
-        async addMissingSchemaFields() {
-          await self.addMissingSchemaFields();
-        }
       }
     };
   },
@@ -236,43 +271,53 @@ module.exports = {
       // Perform the actual migrations. Implementation of
       // the @apostrophecms/migration:migrate task
       async migrate(options) {
-        await self.emit('before');
-        if (self.apos.isNew) {
+        await self.apos.lock.withLock('@apostrophecms/migration:migrate', async () => {
+          if (self.apos.skipMigration === true) {
+            return;
+          }
+
+          await self.emit('before');
+
+          if (self.apos.isNew) {
           // Since the site is brand new (zero documents), we may assume
           // it requires no migrations. Mark them all as "done" but note
           // that they were skipped, just in case we decide that's an issue
           // later
-          const at = new Date();
-          // Just in case the db has no documents but did
-          // start to run migrations on a previous attempt,
-          // which causes an occasional unique key error if not
-          // corrected for here.
-          //
-          // Other migration-related facts that are not migration
-          // names are stored with a leading *, leave them alone
-          await self.db.removeMany({
-            _id: /^[^*]/
-          });
-          await self.db.insertMany(self.migrations.map(migration => ({
-            _id: migration.name,
-            at,
-            skipped: true
-          })));
-        } else {
-          for (const migration of self.migrations) {
-            await self.runOne(migration);
+            const at = new Date();
+            // Just in case the db has no documents but did
+            // start to run migrations on a previous attempt,
+            // which causes an occasional unique key error if not
+            // corrected for here.
+            //
+            // Other migration-related facts that are not migration
+            // names are stored with a leading *, leave them alone
+            await self.db.removeMany({
+              _id: /^[^*]/
+            });
+            await self.db.insertMany(self.migrations.map(migration => ({
+              _id: migration.name,
+              at,
+              skipped: true
+            })));
+          } else {
+            for (const migration of self.migrations) {
+              await self.runOne(migration);
+            }
           }
-        }
-        // In production, this event is emitted only at the end of the migrate
-        // command line task. In dev it is emitted at every startup after the
-        // automatic migration.
-        //
-        // Intentionally emitted regardless of whether the site is new or not.
-        //
-        // This is the right time to park pages, for instance, because the
-        // database is guaranteed to be in a stable state, whether because the
-        // site is new or because migrations ran successfully.
-        await self.emit('after');
+
+          // In production, this event is emitted only at the end of the migrate
+          // command line task. In dev it is emitted at every startup after the
+          // automatic migration.
+          //
+          // Intentionally emitted regardless of whether the site is new or not.
+          //
+          // This is the right time to park pages, for instance, because the
+          // database is guaranteed to be in a stable state, whether because the
+          // site is new or because migrations ran successfully.
+          await self.emit('after');
+
+          await self.emit('requirements');
+        });
       },
       async runOne(migration) {
         const info = await self.db.findOne({ _id: migration.name });
@@ -305,6 +350,12 @@ module.exports = {
         // and automatically detect whether any work
         // must be done
         task: () => {}
+      },
+      'add-missing-schema-fields': {
+        usage: 'Add missing schema fields to existing database documents',
+        task: async () => {
+          await self.addMissingSchemaFields();
+        }
       }
     };
   }

package.json

@@ -5,7 +5,7 @@
   "main": "index.js",
   "scripts": {
     "pretest": "npm run lint",
-    "test": "nyc mocha -t 10000 --ignore=test/assets.js && nyc mocha -t 10000 test/assets.js && mocha -t 1000 test/esm-project/esm.js",
+    "test": "nyc mocha -t 10000 --ignore=test/assets.js && nyc mocha -t 10000 test/add-missing-schema-fields-project/test.js && nyc mocha -t 10000 test/assets.js && mocha -t 1000 test/esm-project/esm.js",
     "eslint": "eslint .",
     "eslint-fix": "npm run eslint -- --fix",
     "i18n": "node scripts/lint-i18n",

test/add-missing-schema-fields-project/app.js

@@ -0,0 +1,4 @@
+import config from './config.js';
+import apostrophe from '../../index.js';
+
+export default apostrophe(config);

test/add-missing-schema-fields-project/config.js

@@ -0,0 +1,13 @@
+export default {
+  root: import.meta,
+  shortName: 'add-missing-schema-fields-project',
+  baseUrl: 'http://localhost:3000',
+  modules: {
+    '@apostrophecms/express': {
+      options: {
+        address: '127.0.0.1'
+      }
+    },
+    product: {}
+  }
+};

test/add-missing-schema-fields-project/modules/product/index.js

@@ -0,0 +1,77 @@
+export default {
+  extend: '@apostrophecms/piece-type',
+  options: {
+    label: 'Product',
+    pluralLabel: 'Products'
+  },
+  fields: {
+    add: {
+      price: {
+        type: 'float',
+        label: 'Price',
+        required: true
+      },
+      description: {
+        type: 'string',
+        label: 'Description',
+        textarea: true,
+        required: true
+      },
+      image: {
+        label: 'Product photo',
+        type: 'area',
+        options: {
+          max: 1,
+          widgets: {
+            '@apostrophecms/image': {}
+          }
+        }
+      },
+      copyright: {
+        type: 'string',
+        label: 'Copyright'
+      }
+    },
+    group: {
+      basics: {
+        label: 'Basics',
+        fields: [ 'title', 'price', 'description', 'image', 'copyright' ]
+        // 👆 'title' is included here because it is in the default `basics`
+        // group for all piece types. Since we are replacing that group, we
+        // include it ourselves.
+      }
+    }
+  },
+  init(self) {
+    self.apos.migration.add('add-first-product', self.addFirstProduct);
+    self.apos.migration.add('add-copyright-notice', self.addCopyrightNotice);
+  },
+  methods(self) {
+    return {
+      async addFirstProduct() {
+        await self.apos.modules.product.insert(
+          self.apos.task.getReq(),
+          {
+            ...self.apos.modules.product.newInstance(),
+            title: 'My first product',
+            price: 10.00,
+            description: 'Product description'
+          }
+        );
+      },
+      async addCopyrightNotice() {
+        await self.apos.migration.eachDoc({
+          type: 'product'
+        }, async (doc) => {
+          if (doc.copyright === undefined) {
+            await self.apos.doc.db.updateOne({
+              _id: doc._id
+            }, {
+              $set: { copyright: '©2024 ApostropheCMS. All rights reserved.' }
+            });
+          }
+        });
+      }
+    };
+  }
+};

test/add-missing-schema-fields-project/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "add-missing-schema-fields-project",
+  "version": "1.0.0",
+  "description": "",
+  "main": "app.js",
+  "type": "module",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "apostrophe": "file:../../."
+  }
+}