tests: add TLS gold tests

AGENTS.md

@@ -123,6 +123,13 @@ regularly).
 using the Proxy Verifier format. This is simpler, more maintainable, and
 parseable by tools.
 
+**Python conventions for test and helper scripts:**
+- Launch Python helpers with `{sys.executable}` rather than a hardcoded `python3`,
+  so the test runs under the same interpreter the harness uses.
+- Prefer f-strings over `str.format()` when building command lines, config lines,
+  and `Testers` expressions.
+- Add type annotations to helper functions.
+
 **For complete details on writing autests, see:**
 - `doc/developer-guide/testing/autests.en.rst` - Comprehensive guide to autest
 - Proxy Verifier format: https://github.com/yahoo/proxy-verifier

tests/gold_tests/tls/tls_flow_control.test.py

@@ -0,0 +1,116 @@
+'''
+Exercise HTTP tunnel flow control (proxy.config.http.flow_control) on a TLS
+client connection: a slow reader makes ATS throttle the origin and unthrottle as
+the buffered data drains. The full response body must still be delivered.
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import sys
+
+Test.Summary = __doc__
+
+
+class TestTlsFlowControl:
+    '''Verify a slow TLS reader under flow control still receives the whole body.'''
+
+    # Comfortably larger than the water marks so the tunnel throttles and has to
+    # unthrottle many times over the transfer.
+    _body_len: int = 8 * 1024 * 1024
+    _high_water: int = 64 * 1024
+    _low_water: int = 32 * 1024
+
+    _server_counter: int = 0
+    _ts_counter: int = 0
+
+    def __init__(self) -> None:
+        '''Declare the test Processes.'''
+        self._server = self._configure_server()
+        self._ts = self._configure_trafficserver()
+
+    def _configure_server(self) -> 'Process':
+        '''Configure the origin server with a large response body.
+
+        :return: The origin server Process.
+        '''
+        server = Test.MakeOriginServer(f'server-{TestTlsFlowControl._server_counter}')
+        TestTlsFlowControl._server_counter += 1
+
+        request_header = {"headers": "GET /obj HTTP/1.1\r\nHost: ex.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
+        response_header = {
+            "headers":
+                "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n"
+                f"Content-Length: {TestTlsFlowControl._body_len}\r\n\r\n",
+            "timestamp": "1469733493.993",
+            "body": "x" * TestTlsFlowControl._body_len
+        }
+        server.addResponse("sessionlog.json", request_header, response_header)
+        return server
+
+    def _configure_trafficserver(self) -> 'Process':
+        '''Configure Traffic Server with HTTP flow control enabled over TLS.
+
+        :return: The Traffic Server Process.
+        '''
+        ts = Test.MakeATSProcess(f'ts-{TestTlsFlowControl._ts_counter}', enable_tls=True, enable_cache=False)
+        TestTlsFlowControl._ts_counter += 1
+
+        ts.addDefaultSSLFiles()
+        ts.Disk.ssl_multicert_yaml.AddLines(
+            """
+ssl_multicert:
+  - dest_ip: "*"
+    ssl_cert_name: server.pem
+    ssl_key_name: server.key
+""".split("\n"))
+        ts.Disk.remap_config.AddLine(f'map / http://127.0.0.1:{self._server.Variables.Port}')
+        ts.Disk.records_config.update(
+            {
+                'proxy.config.ssl.server.cert.path': f'{ts.Variables.SSLDir}',
+                'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}',
+                # Small thresholds on a network sink: the tunnel reenable fires before
+                # the socket write, so the unthrottle depends on the buffer draining.
+                'proxy.config.http.flow_control.enabled': 1,
+                'proxy.config.http.flow_control.high_water': TestTlsFlowControl._high_water,
+                'proxy.config.http.flow_control.low_water': TestTlsFlowControl._low_water,
+            })
+
+        ts.Disk.traffic_out.Content = Testers.ExcludesExpression(
+            "received signal|failed assertion", "ATS must not crash under TLS flow control")
+        ts.Disk.traffic_out.Content += Testers.ExcludesExpression(
+            "AddressSanitizer|use-after-free|runtime error:", "no memory-safety error under TLS flow control")
+        return ts
+
+    def run(self) -> None:
+        '''Configure and run the TestRun.'''
+        tr = Test.AddTestRun("slow TLS reader under flow control must receive the whole body")
+        tr.Processes.Default.StartBefore(self._server)
+        tr.Processes.Default.StartBefore(self._ts)
+        tr.Processes.Default.Command = (
+            f'{sys.executable} {os.path.join(Test.TestDirectory, "tls_flow_control_client.py")} '
+            f'-p {self._ts.Variables.ssl_port} --host ex.test --path /obj '
+            f'--expect-bytes {TestTlsFlowControl._body_len}')
+        tr.Processes.Default.ReturnCode = 0
+        tr.Processes.Default.Streams.All = Testers.ContainsExpression(
+            "RESULT=PASS", "the full body must be delivered under flow control")
+        tr.Processes.Default.Streams.All += Testers.ContainsExpression(
+            f"BODY_BYTES={TestTlsFlowControl._body_len}", "every body byte must arrive (no flow-control stall)")
+        tr.StillRunningAfter = self._ts
+        tr.StillRunningAfter = self._server
+
+
+TestTlsFlowControl().run()

tests/gold_tests/tls/tls_flow_control_client.py

@@ -0,0 +1,100 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Download a large object from ATS over TLS while reading the response body
+slowly, to exercise HTTP tunnel flow control (proxy.config.http.flow_control)
+on a TLS client connection.
+
+A slow reader keeps ATS's client-side write buffer full, so the tunnel
+repeatedly throttles the origin read and must unthrottle as the buffered data
+drains to the client. The whole body must still arrive; a flow-control stall
+(no unthrottle) shows up as a short or timed-out read. This path had no prior
+gold coverage, and the layered TLS VConnection drives the unthrottle through its
+demand-driven write rather than the inherited write-buffer-empty trap, so it is
+worth guarding directly.
+"""
+
+import argparse
+import socket
+import ssl
+import sys
+import time
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-p", "--port", type=int, required=True, help="ATS TLS port")
+    parser.add_argument("--host", default="ex.test", help="Host header / SNI")
+    parser.add_argument("--path", default="/obj", help="request path")
+    parser.add_argument("--expect-bytes", type=int, required=True, help="expected body length")
+    parser.add_argument("--read-size", type=int, default=16 * 1024, help="bytes per recv")
+    parser.add_argument("--read-delay", type=float, default=0.002, help="sleep between recvs (s)")
+    parser.add_argument("--recv-timeout", type=float, default=15.0, help="per-recv socket timeout (s)")
+    parser.add_argument("--deadline", type=float, default=120.0, help="overall wall-clock budget (s)")
+    args = parser.parse_args()
+
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+
+    deadline = time.monotonic() + args.deadline
+    body_received = 0
+    error = ""
+    try:
+        with socket.create_connection(("127.0.0.1", args.port), timeout=args.recv_timeout) as sock:
+            with ctx.wrap_socket(sock, server_hostname=args.host) as ssock:
+                ssock.settimeout(args.recv_timeout)
+                request = (f"GET {args.path} HTTP/1.1\r\nHost: {args.host}\r\n"
+                           "Connection: close\r\n\r\n").encode()
+                ssock.sendall(request)
+
+                # Read until we have the full header block, keeping any body bytes
+                # that arrive in the same recv.
+                buf = b""
+                while b"\r\n\r\n" not in buf:
+                    if time.monotonic() > deadline:
+                        raise TimeoutError("deadline reached reading response headers")
+                    chunk = ssock.recv(args.read_size)
+                    if not chunk:
+                        raise ConnectionError("connection closed before headers complete")
+                    buf += chunk
+                header_blob, _, leftover = buf.partition(b"\r\n\r\n")
+                body_received = len(leftover)
+
+                # Drain the body slowly so ATS's client-side write buffer stays full
+                # and the tunnel relies on the buffer-empty unthrottle.
+                while body_received < args.expect_bytes:
+                    if time.monotonic() > deadline:
+                        raise TimeoutError(f"deadline reached after {body_received} body bytes")
+                    time.sleep(args.read_delay)
+                    chunk = ssock.recv(args.read_size)
+                    if not chunk:
+                        break
+                    body_received += len(chunk)
+    except Exception as exc:  # noqa: BLE001 - any failure is a test signal
+        error = f"{type(exc).__name__}: {exc}"
+
+    passed = error == "" and body_received == args.expect_bytes
+
+    print(f"BODY_BYTES={body_received}")
+    print(f"EXPECT_BYTES={args.expect_bytes}")
+    if error:
+        print(f"ERROR={error}")
+    print(f"RESULT={'PASS' if passed else 'FAIL'}")
+    return 0 if passed else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/gold_tests/tls/tls_origin_open_failed.test.py

@@ -0,0 +1,78 @@
+'''
+Verify that a failed outbound TLS origin connection is surfaced to the client as
+an error (5xx) without crashing ATS.
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import ports
+
+Test.Summary = __doc__
+
+
+class TestOriginOpenFailed:
+    '''Verify a failed outbound TLS connect is surfaced as a 5xx, not a crash.'''
+
+    _ts_counter: int = 0
+
+    def __init__(self) -> None:
+        '''Declare the test Processes.'''
+        self._ts = self._configure_trafficserver()
+
+    def _configure_trafficserver(self) -> 'Process':
+        '''Configure Traffic Server with an https origin whose connect fails.
+
+        :return: The Traffic Server Process.
+        '''
+        ts = Test.MakeATSProcess(f'ts-{TestOriginOpenFailed._ts_counter}')
+        TestOriginOpenFailed._ts_counter += 1
+
+        # Reserve a port for a TLS origin; its liveness is irrelevant since the
+        # source bind below fails before any connect is attempted.
+        ports.get_port(ts, 'origin_port')
+
+        ts.Disk.remap_config.AddLine(f'map http://dead.test/ https://127.0.0.1:{ts.Variables.origin_port}/')
+        ts.Disk.records_config.update(
+            {
+                # Bind every outbound connection to a non-local source address
+                # (RFC 5737 documentation range) so bind() fails synchronously with
+                # EADDRNOTAVAIL, i.e. the connect fails before any TCP/TLS exchange
+                # rather than via a later, routable connect that would be refused
+                # asynchronously.
+                'proxy.config.outgoing_ip_to_bind': '192.0.2.1',
+                'proxy.config.http.connect_attempts_max_retries': 1,
+                'proxy.config.diags.debug.enabled': 1,
+                'proxy.config.diags.debug.tags': 'http|ssl',
+            })
+
+        # Tearing down the failed outbound TLS connect must not crash ATS.
+        ts.Disk.traffic_out.Content = Testers.ExcludesExpression(
+            "received signal|failed assertion", "ATS must not crash on a failed outbound TLS connect")
+        return ts
+
+    def run(self) -> None:
+        '''Configure and run the TestRun.'''
+        tr = Test.AddTestRun("a failed outbound TLS connect is surfaced cleanly")
+        tr.Processes.Default.StartBefore(self._ts)
+        tr.MakeCurlCommand(
+            f'-s -o /dev/null -w "%{{http_code}}" -H "Host: dead.test" http://127.0.0.1:{self._ts.Variables.port}/', ts=self._ts)
+        tr.Processes.Default.ReturnCode = 0
+        # A failed origin connection is surfaced as a 5xx (502 Bad Gateway).
+        tr.Processes.Default.Streams.stdout = Testers.ContainsExpression("50[02]", "a failed origin connect yields a 5xx")
+        tr.StillRunningAfter = self._ts
+
+
+TestOriginOpenFailed().run()