Change LineDecoder to match stdlib splitlines, resulting in significant speed up

httpx/_decoders.py

@@ -267,57 +267,34 @@ def __init__(self) -> None:
         self.buffer = ""
 
     def decode(self, text: str) -> typing.List[str]:
-        lines = []
-
-        if text and self.buffer and self.buffer[-1] == "\r":
-            if text.startswith("\n"):
-                # Handle the case where we have an "\r\n" split across
-                # our previous input, and our new chunk.
-                lines.append(self.buffer[:-1] + "\n")
-                self.buffer = ""
-                text = text[1:]
-            else:
-                # Handle the case where we have "\r" at the end of our
-                # previous input.
-                lines.append(self.buffer[:-1] + "\n")
-                self.buffer = ""
-
-        while text:
-            num_chars = len(text)
-            for idx in range(num_chars):
-                char = text[idx]
-                next_char = None if idx + 1 == num_chars else text[idx + 1]
-                if char == "\n":
-                    lines.append(self.buffer + text[: idx + 1])
-                    self.buffer = ""
-                    text = text[idx + 1 :]
-                    break
-                elif char == "\r" and next_char == "\n":
-                    lines.append(self.buffer + text[:idx] + "\n")
-                    self.buffer = ""
-                    text = text[idx + 2 :]
-                    break
-                elif char == "\r" and next_char is not None:
-                    lines.append(self.buffer + text[:idx] + "\n")
-                    self.buffer = ""
-                    text = text[idx + 1 :]
-                    break
-                elif next_char is None:
-                    self.buffer += text
-                    text = ""
-                    break
+        # See https://docs.python.org/3/library/stdtypes.html#str.splitlines
+        NEWLINE_CHARS = "\n\r\x0b\x0c\x1c\x1d\x1e\x85\u2028\u2029"
+
+        if self.buffer:
+            # If we have some buffered text from the previous pass,
+            # then we include it before handling the input.
+            text = self.buffer + text
+            self.buffer = ""
+
+        if not text:
+            return []
+
+        lines = text.splitlines()
+
+        if text[-1] == "\r":
+            # If the last character is "\r", then we might be on the boundary of
+            # a "\r\n" sequence. We reassemble and buffer the final portion of the input.
+            self.buffer = lines.pop() + "\r"
+        elif text[-1] not in NEWLINE_CHARS:
+            # If the last character is not a newline seperator, then the final portion
+            # from `splitlines()` is incomplete and needs to be buffered for the next
+            # pass.
+            self.buffer = lines.pop()
 
         return lines
 
     def flush(self) -> typing.List[str]:
-        if self.buffer.endswith("\r"):
-            # Handle the case where we had a trailing '\r', which could have
-            # been a '\r\n' pair.
-            lines = [self.buffer[:-1] + "\n"]
-        elif self.buffer:
-            lines = [self.buffer]
-        else:
-            lines = []
+        lines = self.buffer.splitlines()
         self.buffer = ""
         return lines
 

tests/models/test_responses.py

@@ -639,7 +639,7 @@ def test_iter_lines():
         content=b"Hello,\nworld!",
     )
     content = [line for line in response.iter_lines()]
-    assert content == ["Hello,\n", "world!"]
+    assert content == ["Hello,", "world!"]
 
 
 @pytest.mark.anyio
@@ -652,7 +652,7 @@ async def test_aiter_lines():
     content = []
     async for line in response.aiter_lines():
         content.append(line)
-    assert content == ["Hello,\n", "world!"]
+    assert content == ["Hello,", "world!"]
 
 
 def test_sync_streaming_response():

tests/test_decoders.py

@@ -225,69 +225,69 @@ def test_text_decoder_empty_cases():
 def test_line_decoder_nl():
     decoder = LineDecoder()
     assert decoder.decode("") == []
-    assert decoder.decode("a\n\nb\nc") == ["a\n", "\n", "b\n"]
+    assert decoder.decode("a\n\nb\nc") == ["a", "", "b"]
     assert decoder.flush() == ["c"]
 
     decoder = LineDecoder()
     assert decoder.decode("") == []
-    assert decoder.decode("a\n\nb\nc\n") == ["a\n", "\n", "b\n", "c\n"]
+    assert decoder.decode("a\n\nb\nc\n") == ["a", "", "b", "c"]
     assert decoder.flush() == []
 
     # Issue #1033
     decoder = LineDecoder()
     assert decoder.decode("") == []
-    assert decoder.decode("12345\n") == ["12345\n"]
+    assert decoder.decode("12345\n") == ["12345"]
     assert decoder.decode("foo ") == []
     assert decoder.decode("bar ") == []
-    assert decoder.decode("baz\n") == ["foo bar baz\n"]
+    assert decoder.decode("baz\n") == ["foo bar baz"]
     assert decoder.flush() == []
 
 
 def test_line_decoder_cr():
     decoder = LineDecoder()
     assert decoder.decode("") == []
-    assert decoder.decode("a\r\rb\rc") == ["a\n", "\n", "b\n"]
+    assert decoder.decode("a\r\rb\rc") == ["a", "", "b"]
     assert decoder.flush() == ["c"]
 
     decoder = LineDecoder()
     assert decoder.decode("") == []
-    assert decoder.decode("a\r\rb\rc\r") == ["a\n", "\n", "b\n"]
-    assert decoder.flush() == ["c\n"]
+    assert decoder.decode("a\r\rb\rc\r") == ["a", "", "b"]
+    assert decoder.flush() == ["c"]
 
     # Issue #1033
     decoder = LineDecoder()
     assert decoder.decode("") == []
     assert decoder.decode("12345\r") == []
-    assert decoder.decode("foo ") == ["12345\n"]
+    assert decoder.decode("foo ") == ["12345"]
     assert decoder.decode("bar ") == []
     assert decoder.decode("baz\r") == []
-    assert decoder.flush() == ["foo bar baz\n"]
+    assert decoder.flush() == ["foo bar baz"]
 
 
 def test_line_decoder_crnl():
     decoder = LineDecoder()
     assert decoder.decode("") == []
-    assert decoder.decode("a\r\n\r\nb\r\nc") == ["a\n", "\n", "b\n"]
+    assert decoder.decode("a\r\n\r\nb\r\nc") == ["a", "", "b"]
     assert decoder.flush() == ["c"]
 
     decoder = LineDecoder()
     assert decoder.decode("") == []
-    assert decoder.decode("a\r\n\r\nb\r\nc\r\n") == ["a\n", "\n", "b\n", "c\n"]
+    assert decoder.decode("a\r\n\r\nb\r\nc\r\n") == ["a", "", "b", "c"]
     assert decoder.flush() == []
 
     decoder = LineDecoder()
     assert decoder.decode("") == []
     assert decoder.decode("a\r") == []
-    assert decoder.decode("\n\r\nb\r\nc") == ["a\n", "\n", "b\n"]
+    assert decoder.decode("\n\r\nb\r\nc") == ["a", "", "b"]
     assert decoder.flush() == ["c"]
 
     # Issue #1033
     decoder = LineDecoder()
     assert decoder.decode("") == []
-    assert decoder.decode("12345\r\n") == ["12345\n"]
+    assert decoder.decode("12345\r\n") == ["12345"]
     assert decoder.decode("foo ") == []
     assert decoder.decode("bar ") == []
-    assert decoder.decode("baz\r\n") == ["foo bar baz\n"]
+    assert decoder.decode("baz\r\n") == ["foo bar baz"]
     assert decoder.flush() == []