PRO-8154: Strip Latin accents (#5137)

* Add stripUrlAccents option to control accent stripping in slugs and attachment names

Author: myovchev

.gitignore

@@ -3,6 +3,7 @@
 # Ignore unison sync files
 .unison*
 package-lock.json
+pnpm-lock.yaml
 npm-debug.log
 *.DS_Store
 *.npmignore

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Adds
 
+* Adds `stripUrlAccents` option in `@apostrophecms/i18n` module to globally control whether accents are stripped from URLs. When set to `true`, all URLs (slugs) will have accents from Latin characters removed on document creation and updates. No existing documents are modified automatically; this only affects new or updated documents. A new task `node app @apostrophecms/i18n:strip-slug-accents` is provided to update existing document slugs in the database when needed.
 * Add `@apostrophecms/migration:add-missing-schema-fields` task. This task does not run database migrations.
 * Translation strings added for the layout- and layout-column-widgets.
 * New `box` schema field type

modules/@apostrophecms/doc/index.js

@@ -205,7 +205,11 @@ module.exports = {
       '@apostrophecms/doc-type:beforePublish': {
         testPermissions(req, info) {
           if (info.options.permissions !== false) {
-            if (!self.apos.permission.can(req, info.options.autopublishing ? 'edit' : 'publish', info.draft)) {
+            if (!self.apos.permission.can(
+              req,
+              info.options.autopublishing ? 'edit' : 'publish',
+              info.draft
+            )) {
               throw self.apos.error('forbidden');
             }
           }
@@ -217,7 +221,11 @@ module.exports = {
           manager.ensureSlug(doc);
           _.each(manager.schema, function (field) {
             if (field.sortify) {
-              doc[field.name + 'Sortified'] = self.apos.util.sortify(doc[field.name] ? doc[field.name] : '');
+              doc[field.name + 'Sortified'] = self.apos.util.sortify(
+                doc[field.name]
+                  ? doc[field.name]
+                  : ''
+              );
             }
           });
           if (options.setUpdatedAtAndBy !== false) {

modules/@apostrophecms/i18n/index.js

@@ -70,7 +70,9 @@ module.exports = {
     i18n: {
       ns: 'apostrophe',
       browser: true
-    }
+    },
+    // If true, slugifying will strip accents from Latin characters
+    stripUrlAccents: false
   },
   async init(self) {
     self.defaultNamespace = 'default';
@@ -677,7 +679,8 @@ module.exports = {
           debug: self.debug,
           show: self.show,
           action: self.action,
-          crossDomainClipboard: req.session && req.session.aposCrossDomainClipboard
+          crossDomainClipboard: req.session && req.session.aposCrossDomainClipboard,
+          stripUrlAccents: self.options.stripUrlAccents
         };
         if (req.session && req.session.aposCrossDomainClipboard) {
           req.session.aposCrossDomainClipboard = null;
@@ -734,6 +737,9 @@ module.exports = {
         }
         return locale;
       },
+      shouldStripAccents() {
+        return self.options.stripUrlAccents === true;
+      },
       addLocalizeModal() {
         self.apos.modal.add(
           `${self.__meta.name}:localize`,
@@ -1251,6 +1257,38 @@ module.exports = {
             console.log(`Due to conflicts, kept ${kept} documents from ${keep}`);
           }
         }
+      },
+      'strip-slug-accents': {
+        usage: 'Remove Latin accent characters from all document slugs. Usage: node app @apostrophecms/i18n:strip-slug-accents',
+        async task() {
+          let docChanged = 0;
+
+          await self.apos.migration.eachDoc({}, 5, async doc => {
+            const slug = doc.slug;
+            const req = self.apos.task.getAdminReq({
+              locale: doc.aposLocale?.split(':')[0] || self.defaultLocale
+            });
+            if (!self.shouldStripAccents()) {
+              return;
+            }
+
+            doc.slug = _.deburr(doc.slug);
+            if (slug === doc.slug) {
+              return;
+            }
+            const manager = self.apos.doc.getManager(doc.type);
+            if (!manager) {
+              return;
+            }
+            await manager.update(req, doc, { permissions: false });
+            docChanged++;
+            self.apos.util.log(`Updated doc [${req.locale}] "${slug}" -> "${doc.slug}"`);
+          });
+
+          self.apos.util.log(
+            `Updated ${docChanged} document slug(s).`
+          );
+        }
       }
     };
   }

modules/@apostrophecms/schema/ui/apos/logic/AposInputSlug.js

@@ -3,6 +3,7 @@
 // errors.
 import { klona } from 'klona';
 import sluggo from 'sluggo';
+import { deburr } from 'lodash';
 import AposInputMixin from 'Modules/@apostrophecms/schema/mixins/AposInputMixin';
 import { debounceAsync } from 'Modules/@apostrophecms/ui/utils';
 
@@ -48,6 +49,9 @@ export default {
     },
     localePrefix() {
       return this.field.page && apos.i18n.locales[apos.i18n.locale].prefix;
+    },
+    stripAccents() {
+      return apos.i18n.stripUrlAccents === true;
     }
   },
   watch: {
@@ -192,6 +196,9 @@ export default {
       }
 
       let slug = sluggo(s, options);
+      if (this.stripAccents) {
+        slug = deburr(slug);
+      }
       if (preserveDash) {
         slug += '-';
       }

modules/@apostrophecms/util/index.js

@@ -291,8 +291,18 @@ module.exports = {
       // ONE punctuation character normally forbidden in slugs may
       // optionally be permitted by specifying it via options.allow.
       // The separator may be changed via options.separator.
+      // By default, the i18n.options.stripUrlAccents option is honored;
+      // having stripAccents passed as an option takes precedence.
       slugify(s, options) {
-        return require('sluggo')(s, options);
+        const { stripAccents, ...opts } = options || {};
+        const slug = require('sluggo')(s, opts);
+        const shouldStripAccents = (typeof stripAccents !== 'undefined')
+          ? stripAccents
+          : self.apos.i18n.options.stripUrlAccents;
+        if (shouldStripAccents) {
+          return _.deburr(slug);
+        }
+        return slug;
       },
       // Returns a string that, when used for indexes, behaves
       // similarly to MySQL's default behavior for sorting, plus a little
@@ -938,7 +948,8 @@ module.exports = {
       // ONE punctuation character normally forbidden in slugs may
       // optionally be permitted by specifying it via options.allow.
       // The separator may be changed via options.separator.
-
+      // By default, the i18n.options.stripUrlAccents option is honored;
+      // having stripAccents passed as an option takes precedence.
       slugify: function(string, options) {
         return self.slugify(string, options);
       },