Merge pull request #429 from willkg/422-amp

Convert & to & as a Characters token

Author: willkg

bleach/sanitizer.py

@@ -395,7 +395,18 @@ def sanitize_characters(self, token):
             if part.startswith('&'):
                 entity = html5lib_shim.match_entity(part)
                 if entity is not None:
-                    new_tokens.append({'type': 'Entity', 'name': entity})
+                    if entity == 'amp':
+                        # LinkifyFilter can't match urls across token boundaries
+                        # which is problematic with & since that shows up in
+                        # querystrings all the time. This special-cases &
+                        # and converts it to a & and sticks it in as a
+                        # Characters token. It'll get merged with surrounding
+                        # tokens in the BleachSanitizerfilter.__iter__ and
+                        # escaped in the serializer.
+                        new_tokens.append({'type': 'Characters', 'data': '&'})
+                    else:
+                        new_tokens.append({'type': 'Entity', 'name': entity})
+
                     # Length of the entity plus 2--one for & at the beginning
                     # and and one for ; at the end
                     remainder = part[len(entity) + 2:]

tests/test_linkify.py

@@ -694,6 +694,10 @@ def test_only_text_is_linkified(self):
         'http://example.com?b=1&c=2',
         '<a href="http://example.com?b=1&amp;c=2">http://example.com?b=1&amp;c=2</a>'
     ),
+    (
+        'http://example.com?b=1&amp;c=2',
+        '<a href="http://example.com?b=1&amp;c=2">http://example.com?b=1&amp;c=2</a>'
+    ),
     (
         'link: https://example.com/watch#anchor',
         'link: <a href="https://example.com/watch#anchor">https://example.com/watch#anchor</a>'