feat: add emStrongMask hook (#3749)

* Add hook to mask input for em/strong processing in extension

* Use passthrough hook instead of tokenizer-specific hook

* Remove deprecated items

* Ensure passthrough hook does not create promise

* Add test for combining hooks and async

Author: jonatanschroeder

docs/USING_PRO.md

@@ -263,6 +263,7 @@ Hooks are methods that hook into some part of marked. The following hooks are av
 | `preprocess(markdown: string): string` | Process markdown before sending it to marked. |
 | `postprocess(html: string): string` | Process html after marked has finished parsing. |
 | `processAllTokens(tokens: Token[]): Token[]` | Process all tokens before walk tokens. |
+| `emStrongMask(src: string): string` | Mask part of the content that should not be interpreted as Markdown em/strong delimiters. |
 | `provideLexer(): (src: string, options?: MarkedOptions) => Token[]` | Provide function to tokenize markdown. |
 | `provideParser(): (tokens: Token[], options?: MarkedOptions) => string` | Provide function to parse tokens. |
 
@@ -368,6 +369,27 @@ console.log(marked.parse(`
 <p><a href="http://example.com">test link</a></p>
 ```
 
+**Example:** Mask underline characters inside Mathjax content delimited by `$`
+
+```js
+import { marked } from 'marked';
+
+// Override function
+function emStrongMask(src) {
+  return src.replace(/\$([^$]+)\$/g, (match) => `[${'a'.repeat(match.length - 2)}]`);
+}
+
+marked.use({ hooks: { emStrongMask } });
+
+console.log(marked.parse(`_The formula is $a_ b=c_ d$._`));
+```
+
+**Output:**
+
+```html
+<p><em>The formula is $a_ b=c_ d$.</em></p>
+```
+
 ***
 
 <h2 id="extensions">Custom Extensions : <code>extensions</code></h2>

src/Hooks.ts

@@ -16,6 +16,13 @@ export class _Hooks<ParserOutput = string, RendererOutput = string> {
     'preprocess',
     'postprocess',
     'processAllTokens',
+    'emStrongMask',
+  ]);
+
+  static passThroughHooksRespectAsync = new Set([
+    'preprocess',
+    'postprocess',
+    'processAllTokens',
   ]);
 
   /**
@@ -39,6 +46,13 @@ export class _Hooks<ParserOutput = string, RendererOutput = string> {
     return tokens;
   }
 
+  /**
+   * Mask contents that should not be interpreted as em/strong delimiters
+   */
+  emStrongMask(src: string) {
+    return src;
+  }
+
   /**
    * Provide function to tokenize markdown
    */

src/Instance.ts

@@ -205,7 +205,7 @@ export class Marked<ParserOutput = string, RendererOutput = string> {
           if (_Hooks.passThroughHooks.has(prop)) {
             // @ts-expect-error cannot type hook function dynamically
             hooks[hooksProp] = (arg: unknown) => {
-              if (this.defaults.async) {
+              if (this.defaults.async && _Hooks.passThroughHooksRespectAsync.has(prop)) {
                 return Promise.resolve(hooksFunc.call(hooks, arg)).then(ret => {
                   return prevHook.call(hooks, ret);
                 });

src/Lexer.ts

@@ -324,6 +324,9 @@ export class _Lexer<ParserOutput = string, RendererOutput = string> {
       maskedSrc = maskedSrc.slice(0, match.index) + '[' + 'a'.repeat(match[0].length - 2) + ']' + maskedSrc.slice(this.tokenizer.rules.inline.blockSkip.lastIndex);
     }
 
+    // Mask out blocks from extensions
+    maskedSrc = this.options.hooks?.emStrongMask?.call({ lexer: this }, maskedSrc) ?? maskedSrc;
+
     let keepPrevChar = false;
     let prevChar = '';
     while (src) {

src/MarkedOptions.ts

@@ -77,6 +77,7 @@ export interface MarkedExtension<ParserOutput = string, RendererOutput = string>
    * preprocess is called to process markdown before sending it to marked.
    * processAllTokens is called with the TokensList before walkTokens.
    * postprocess is called to process html after marked has finished parsing.
+   * emStrongMask is called to mask contents that should not be interpreted as em/strong delimiters.
    * provideLexer is called to provide a function to tokenize markdown.
    * provideParser is called to provide a function to parse tokens.
    */

test/unit/marked.test.js

@@ -166,6 +166,46 @@ describe('marked unit', () => {
       assert.strictEqual(html, '<p>Not Underlined <u>Underlined</u> Not Underlined</p>\n');
     });
 
+    it('should ignore em termination characters when emStrongMask hook is in place', () => {
+      const underline = {
+        name: 'underline',
+        level: 'inline',
+        start(src) { return src.indexOf('='); },
+        tokenizer(src) {
+          const rule = /^=([^=]+)=/;
+          const match = rule.exec(src);
+          if (match) {
+            return {
+              type: 'underline',
+              raw: match[0], // This is the text that you want your token to consume from the source
+              text: match[1].trim(), // You can add additional properties to your tokens to pass along to the renderer
+            };
+          }
+        },
+        renderer(token) {
+          return `<u>${token.text}</u>`;
+        },
+      };
+      marked.use({
+        hooks: {
+          // Underline takes priority over emphasis in this example, to mask emphasis markers inside underline tags
+          emStrongMask: (src) => src.replace(/=([^=]+)=/g, (match) => `[${'a'.repeat(match.length - 2)}]`),
+        },
+        extensions: [underline],
+      });
+      const html = marked.parse('*Not Underlined =Underlined* with *asterisk= Not Underlined*');
+      assert.strictEqual(html, '<p><em>Not Underlined <u>Underlined* with *asterisk</u> Not Underlined</em></p>\n');
+    });
+
+    it('should combine multiple emStrongMask hooks', () => {
+      const maskEqualSign = (src) => src.replace(/=([^=]+)=/g, (match) => `[${'a'.repeat(match.length - 2)}]`);
+      const maskDollarSign = (src) => src.replace(/\$([^$]+)\$/g, (match) => `[${'b'.repeat(match.length - 2)}]`);
+      marked.use({ hooks: { emStrongMask: maskEqualSign } });
+      marked.use({ hooks: { emStrongMask: maskDollarSign } });
+      const html = marked.parse('*Before $dollar * dollar$ =equal * equal= after*');
+      assert.strictEqual(html, '<p><em>Before $dollar * dollar$ =equal * equal= after</em></p>\n');
+    });
+
     it('should handle interacting block and inline extensions', () => {
       const descriptionlist = {
         name: 'descriptionList',
@@ -650,6 +690,38 @@ used extension2 walked</p>
       assert.throws(() => marked.parse('test', { async: false }));
     });
 
+    it('should ignore em termination characters when emStrongMask hook is in place in an async context', async() => {
+      const underline = {
+        name: 'underline',
+        level: 'inline',
+        start(src) { return src.indexOf('='); },
+        tokenizer(src) {
+          const rule = /^=([^=]+)=/;
+          const match = rule.exec(src);
+          if (match) {
+            return {
+              type: 'underline',
+              raw: match[0], // This is the text that you want your token to consume from the source
+              text: match[1].trim(), // You can add additional properties to your tokens to pass along to the renderer
+            };
+          }
+        },
+        renderer(token) {
+          return `<u>${token.text}</u>`;
+        },
+      };
+      marked.use({
+        hooks: {
+          // Underline takes priority over emphasis in this example, to mask emphasis markers inside underline tags
+          emStrongMask: (src) => src.replace(/=([^=]+)=/g, (match) => `[${'a'.repeat(match.length - 2)}]`),
+        },
+        extensions: [underline],
+        async: true,
+      });
+      const html = await marked.parse('*Not Underlined =Underlined* with *asterisk= Not Underlined*');
+      assert.strictEqual(html, '<p><em>Not Underlined <u>Underlined* with *asterisk</u> Not Underlined</em></p>\n');
+    });
+
     it('should allow deleting/editing tokens', () => {
       const styleTags = {
         extensions: [{