From f9e3f195ad06ef3ad5a09b982d74f1402dfb0771 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 5 Oct 2025 22:42:47 +0000 Subject: [PATCH 1/7] Add cross-platform support for Autobahn test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit enables the Autobahn WebSocket protocol compliance test suite to run on both Docker Desktop (Mac/Windows) and Docker-in-Docker (Linux) environments. Changes: - Auto-detect platform and use appropriate Docker networking - Mac/Windows: Uses host.docker.internal with port mapping - Linux: Uses --network=host with localhost - Add fuzzingclient-linux.json config for Linux environments - Add DOCKER_AUTOBAHN_SETUP.md with installation instructions Test Results (517 total tests): - ✅ 294 passed (56.9%) - ✅ 0 failed - 📝 4 non-strict (acceptable deviations) - 📝 3 informational (expected behavior) - 📝 216 unimplemented (optional compression extensions) All core RFC 6455 protocol tests pass successfully. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- DOCKER_AUTOBAHN_SETUP.md | 94 +++++++++++++++++++ test/autobahn/config/fuzzingclient-linux.json | 17 ++++ test/autobahn/run-wstest.js | 12 ++- 3 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 DOCKER_AUTOBAHN_SETUP.md create mode 100644 test/autobahn/config/fuzzingclient-linux.json diff --git a/DOCKER_AUTOBAHN_SETUP.md b/DOCKER_AUTOBAHN_SETUP.md new file mode 100644 index 00000000..65f76a5f --- /dev/null +++ b/DOCKER_AUTOBAHN_SETUP.md @@ -0,0 +1,94 @@ +# Docker Container Setup: Autobahn Test Suite + +## Overview + +This document provides instructions for modifying the Dockerfile to include the Autobahn WebSocket Protocol Test Suite, enabling RFC 6455 compliance testing without requiring Docker-in-Docker. + +## Required Dockerfile Modifications + +### 1. Install Python pip and virtualenv + +Add the following to your Dockerfile to install pip3 and venv support: + +```dockerfile +RUN apt-get update && \ + apt-get install -y python3-pip python3-venv && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +``` + +### 2. Install Autobahn Test Suite + +Create a Python virtual environment and install the Autobahn Test Suite: + +```dockerfile +RUN python3 -m venv /home/ubuntu/wstest-env && \ + /home/ubuntu/wstest-env/bin/pip install --no-cache-dir autobahntestsuite +``` + +### 3. Set Ownership (if needed) + +If your container runs as a non-root user (e.g., `ubuntu`), ensure proper ownership: + +```dockerfile +RUN chown -R ubuntu:ubuntu /home/ubuntu/wstest-env +``` + +## Complete Dockerfile Section + +Here's a complete section you can add to your Dockerfile: + +```dockerfile +# Install Autobahn WebSocket Test Suite +RUN apt-get update && \ + apt-get install -y python3-pip python3-venv && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN python3 -m venv /home/ubuntu/wstest-env && \ + /home/ubuntu/wstest-env/bin/pip install --no-cache-dir autobahntestsuite && \ + chown -R ubuntu:ubuntu /home/ubuntu/wstest-env +``` + +## Verification + +After building the container, verify the installation: + +```bash +# Check Python is available +python3 --version + +# Check pip is available +/home/ubuntu/wstest-env/bin/pip --version + +# Check wstest is installed +/home/ubuntu/wstest-env/bin/wstest --version + +# Expected output: Shows Autobahn Test Suite version (e.g., 0.8.x) +``` + +## Environment Details + +- **Python Version Required:** Python 3.x (Python 3.12.3 confirmed working) +- **Installation Method:** pip via Python virtualenv +- **Install Location:** `/home/ubuntu/wstest-env/` +- **Binary Location:** `/home/ubuntu/wstest-env/bin/wstest` +- **Package Name:** `autobahntestsuite` (PyPI) + +## Size Considerations + +- Python 3 + pip: ~50-100 MB +- Autobahn Test Suite + dependencies: ~30-50 MB +- Total additional space: ~100-150 MB + +## Notes + +- The virtual environment approach isolates dependencies and prevents conflicts +- `--no-cache-dir` flag reduces image size by not caching pip downloads +- Cleaning apt cache (`rm -rf /var/lib/apt/lists/*`) further reduces image size +- The test suite will be available at `/home/ubuntu/wstest-env/bin/wstest` +- No Docker-in-Docker required - tests run natively in the container + +## Testing After Installation + +Once the container is built and running, the WebSocket-Node test suite will automatically detect and use the installed `wstest` binary to run Autobahn protocol compliance tests. diff --git a/test/autobahn/config/fuzzingclient-linux.json b/test/autobahn/config/fuzzingclient-linux.json new file mode 100644 index 00000000..0859770c --- /dev/null +++ b/test/autobahn/config/fuzzingclient-linux.json @@ -0,0 +1,17 @@ + +{ + "options": {"failByDrop": false}, + "outdir": "./reports/servers", + + "servers": [ + { + "agent": "WebSocket-Node 1.0.27", + "url": "ws://localhost:8080", + "options": {"version": 18} + } + ], + + "cases": ["*"], + "exclude-cases": [], + "exclude-agent-cases": {} +} diff --git a/test/autobahn/run-wstest.js b/test/autobahn/run-wstest.js index 765434a1..abea6a3b 100755 --- a/test/autobahn/run-wstest.js +++ b/test/autobahn/run-wstest.js @@ -106,16 +106,22 @@ class AutobahnTestRunner { runAutobahnTests() { return new Promise((resolve, reject) => { console.log('🐳 Starting Autobahn test suite with Docker...'); - + + // Detect platform and use appropriate config and networking + const isLinux = process.platform === 'linux'; + const configFile = isLinux ? 'fuzzingclient-linux.json' : 'fuzzingclient.json'; + + console.log(` Platform: ${process.platform}, using config: ${configFile}`); + const dockerArgs = [ 'run', '--rm', + ...(isLinux ? ['--network=host'] : ['-p', '9001:9001']), '-v', `${process.cwd()}/config:/config`, '-v', `${process.cwd()}/reports:/reports`, - '-p', '9001:9001', '--name', 'fuzzingclient', 'crossbario/autobahn-testsuite', - 'wstest', '-m', 'fuzzingclient', '--spec', '/config/fuzzingclient.json' + 'wstest', '-m', 'fuzzingclient', '--spec', `/config/${configFile}` ]; this.dockerProcess = spawn('docker', dockerArgs, { From d87a2543eaed7c6a7a494a8c922cfc77624b5311 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 5 Oct 2025 23:15:14 +0000 Subject: [PATCH 2/7] Remove obsolete DOCKER_AUTOBAHN_SETUP.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This file was created for native Python installation instructions, but is no longer needed since we're using Docker-in-Docker. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- DOCKER_AUTOBAHN_SETUP.md | 94 ---------------------------------------- 1 file changed, 94 deletions(-) delete mode 100644 DOCKER_AUTOBAHN_SETUP.md diff --git a/DOCKER_AUTOBAHN_SETUP.md b/DOCKER_AUTOBAHN_SETUP.md deleted file mode 100644 index 65f76a5f..00000000 --- a/DOCKER_AUTOBAHN_SETUP.md +++ /dev/null @@ -1,94 +0,0 @@ -# Docker Container Setup: Autobahn Test Suite - -## Overview - -This document provides instructions for modifying the Dockerfile to include the Autobahn WebSocket Protocol Test Suite, enabling RFC 6455 compliance testing without requiring Docker-in-Docker. - -## Required Dockerfile Modifications - -### 1. Install Python pip and virtualenv - -Add the following to your Dockerfile to install pip3 and venv support: - -```dockerfile -RUN apt-get update && \ - apt-get install -y python3-pip python3-venv && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* -``` - -### 2. Install Autobahn Test Suite - -Create a Python virtual environment and install the Autobahn Test Suite: - -```dockerfile -RUN python3 -m venv /home/ubuntu/wstest-env && \ - /home/ubuntu/wstest-env/bin/pip install --no-cache-dir autobahntestsuite -``` - -### 3. Set Ownership (if needed) - -If your container runs as a non-root user (e.g., `ubuntu`), ensure proper ownership: - -```dockerfile -RUN chown -R ubuntu:ubuntu /home/ubuntu/wstest-env -``` - -## Complete Dockerfile Section - -Here's a complete section you can add to your Dockerfile: - -```dockerfile -# Install Autobahn WebSocket Test Suite -RUN apt-get update && \ - apt-get install -y python3-pip python3-venv && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN python3 -m venv /home/ubuntu/wstest-env && \ - /home/ubuntu/wstest-env/bin/pip install --no-cache-dir autobahntestsuite && \ - chown -R ubuntu:ubuntu /home/ubuntu/wstest-env -``` - -## Verification - -After building the container, verify the installation: - -```bash -# Check Python is available -python3 --version - -# Check pip is available -/home/ubuntu/wstest-env/bin/pip --version - -# Check wstest is installed -/home/ubuntu/wstest-env/bin/wstest --version - -# Expected output: Shows Autobahn Test Suite version (e.g., 0.8.x) -``` - -## Environment Details - -- **Python Version Required:** Python 3.x (Python 3.12.3 confirmed working) -- **Installation Method:** pip via Python virtualenv -- **Install Location:** `/home/ubuntu/wstest-env/` -- **Binary Location:** `/home/ubuntu/wstest-env/bin/wstest` -- **Package Name:** `autobahntestsuite` (PyPI) - -## Size Considerations - -- Python 3 + pip: ~50-100 MB -- Autobahn Test Suite + dependencies: ~30-50 MB -- Total additional space: ~100-150 MB - -## Notes - -- The virtual environment approach isolates dependencies and prevents conflicts -- `--no-cache-dir` flag reduces image size by not caching pip downloads -- Cleaning apt cache (`rm -rf /var/lib/apt/lists/*`) further reduces image size -- The test suite will be available at `/home/ubuntu/wstest-env/bin/wstest` -- No Docker-in-Docker required - tests run natively in the container - -## Testing After Installation - -Once the container is built and running, the WebSocket-Node test suite will automatically detect and use the installed `wstest` binary to run Autobahn protocol compliance tests. From b7064247ab6b11a12beff318613da19b27864877 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 5 Oct 2025 23:17:02 +0000 Subject: [PATCH 3/7] Add Autobahn protocol tests to GitHub Actions workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workflow now runs the comprehensive Autobahn WebSocket protocol compliance test suite (517 tests) in addition to the existing unit tests. Changes: - Pull crossbario/autobahn-testsuite Docker image - Run full Autobahn test suite via run-wstest.js - Tests will fail CI if any protocol compliance issues are detected The parse-results.js script exits with code 1 on test failures, which will cause the GitHub Actions workflow to fail appropriately. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/websocket-tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/websocket-tests.yml b/.github/workflows/websocket-tests.yml index f98d3956..fd022cae 100644 --- a/.github/workflows/websocket-tests.yml +++ b/.github/workflows/websocket-tests.yml @@ -20,3 +20,9 @@ jobs: - run: pnpm run lint - run: pnpm run test + + - name: Pull Autobahn Test Suite Docker image + run: docker pull crossbario/autobahn-testsuite + + - name: Run Autobahn WebSocket Protocol Tests + run: cd test/autobahn && node run-wstest.js From 9776a929fad56345f81cd771c74282d711c8265e Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 5 Oct 2025 23:21:10 +0000 Subject: [PATCH 4/7] Fix exit code handling for Autobahn test failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test runner was intercepting process.exit() but not properly propagating failure exit codes, causing CI to pass even with test failures. Changes: - parse-results.js: Return summary object for programmatic access - run-wstest.js: Capture exit code and check summary.failed - Exit with code 1 if there are any test failures - Exit with code 1 if results file is missing Tested with mock results: - ✅ Exit code 1 with failures - ✅ Exit code 0 with all passing This ensures GitHub Actions CI will fail appropriately when protocol compliance tests fail. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- test/autobahn/parse-results.js | 4 +++- test/autobahn/run-wstest.js | 28 +++++++++++++++++++--------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/test/autobahn/parse-results.js b/test/autobahn/parse-results.js index 63850b52..7914e47b 100755 --- a/test/autobahn/parse-results.js +++ b/test/autobahn/parse-results.js @@ -140,7 +140,7 @@ function parseResults() { } console.log('\n'); - + // Exit with error code if there are actual failures if (summary.failed > 0) { console.error(`❌ ${summary.failed} test(s) failed!`); @@ -148,6 +148,8 @@ function parseResults() { } else { console.log(`✅ All tests passed! (${summary.ok} OK, ${summary.nonStrict} non-strict, ${summary.informational} informational, ${summary.unimplemented} unimplemented)`); } + + return summary; } if (require.main === module) { diff --git a/test/autobahn/run-wstest.js b/test/autobahn/run-wstest.js index abea6a3b..621c6659 100755 --- a/test/autobahn/run-wstest.js +++ b/test/autobahn/run-wstest.js @@ -163,26 +163,36 @@ class AutobahnTestRunner { parseAndDisplayResults() { console.log('📊 Parsing test results...\n'); - + const resultsPath = path.join(__dirname, 'reports', 'servers', 'index.json'); - + if (!fs.existsSync(resultsPath)) { console.error('❌ Results file not found. Tests may not have completed properly.'); - return; + process.exit(1); } try { const originalProcessExit = process.exit; - // Prevent parseResults from exiting the process - process.exit = () => {}; - - parseResults(); - + let exitCode = 0; + + // Intercept process.exit to capture the exit code + process.exit = (code) => { + exitCode = code || 0; + }; + + const summary = parseResults(); + // Restore original function process.exit = originalProcessExit; - + + // Exit with appropriate code if there were failures + if (exitCode !== 0 || (summary && summary.failed > 0)) { + process.exit(1); + } + } catch (error) { console.error('❌ Failed to parse results:', error.message); + process.exit(1); } } From dda9d49dfbd69af163adda21ba3511afc1a65963 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 5 Oct 2025 23:24:32 +0000 Subject: [PATCH 5/7] Improve pass rate calculation to exclude unimplemented tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the pass rate included unimplemented tests (optional WebSocket extensions) in the denominator, showing 56.9% (294/517). This was misleading since unimplemented extensions are not failures. Changes: - Show both 'Total tests' and 'Implemented tests' counts - Calculate pass rate based only on implemented tests - Label as 'Pass rate (implemented)' for clarity Results with current implementation: - Before: 56.9% (294/517 including unimplemented) - After: 97.7% (294/301 excluding unimplemented) This gives a more accurate representation of protocol compliance for the implemented features. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- test/autobahn/parse-results.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/autobahn/parse-results.js b/test/autobahn/parse-results.js index 7914e47b..ef25a789 100755 --- a/test/autobahn/parse-results.js +++ b/test/autobahn/parse-results.js @@ -86,14 +86,18 @@ function parseResults() { // Print summary console.log('Test Summary:'); console.log(` Total tests: ${summary.total}`); + console.log(` Implemented tests: ${summary.total - summary.unimplemented}`); console.log(` Passed (OK): ${summary.ok}`); console.log(` Failed: ${summary.failed}`); console.log(` Non-Strict: ${summary.nonStrict}`); console.log(` Informational: ${summary.informational}`); console.log(` Unimplemented: ${summary.unimplemented}`); - - const passRate = ((summary.ok / summary.total) * 100).toFixed(1); - console.log(` Pass rate: ${passRate}%`); + + const implementedTests = summary.total - summary.unimplemented; + const passRate = implementedTests > 0 + ? ((summary.ok / implementedTests) * 100).toFixed(1) + : '0.0'; + console.log(` Pass rate (implemented): ${passRate}%`); // Print failed tests if any if (summary.failedTests.length > 0) { From 360aff7cceb3150ef1c399b9c049f2c7ce988135 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 5 Oct 2025 23:25:06 +0000 Subject: [PATCH 6/7] Clarify test output: required vs optional protocol features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updated terminology to accurately reflect that unimplemented tests are for optional WebSocket protocol features (like compression extensions), not missing implementation of required features. Changes: - 'Implemented tests' → 'Required tests' - Added 'Optional tests' line to show count clearly - 'Pass rate (implemented)' → 'Pass rate (required)' - 'UNIMPLEMENTED TESTS' → 'OPTIONAL FEATURES NOT IMPLEMENTED' Example output: Total tests: 517 Required tests: 301 Optional tests: 216 Pass rate (required): 97.7% This makes it clear that 97.7% of required protocol functionality passes, and 216 tests are for optional extensions not implemented. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- test/autobahn/parse-results.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/autobahn/parse-results.js b/test/autobahn/parse-results.js index ef25a789..dc1c53ff 100755 --- a/test/autobahn/parse-results.js +++ b/test/autobahn/parse-results.js @@ -86,18 +86,18 @@ function parseResults() { // Print summary console.log('Test Summary:'); console.log(` Total tests: ${summary.total}`); - console.log(` Implemented tests: ${summary.total - summary.unimplemented}`); + console.log(` Required tests: ${summary.total - summary.unimplemented}`); + console.log(` Optional tests: ${summary.unimplemented}`); console.log(` Passed (OK): ${summary.ok}`); console.log(` Failed: ${summary.failed}`); console.log(` Non-Strict: ${summary.nonStrict}`); console.log(` Informational: ${summary.informational}`); - console.log(` Unimplemented: ${summary.unimplemented}`); - const implementedTests = summary.total - summary.unimplemented; - const passRate = implementedTests > 0 - ? ((summary.ok / implementedTests) * 100).toFixed(1) + const requiredTests = summary.total - summary.unimplemented; + const passRate = requiredTests > 0 + ? ((summary.ok / requiredTests) * 100).toFixed(1) : '0.0'; - console.log(` Pass rate (implemented): ${passRate}%`); + console.log(` Pass rate (required): ${passRate}%`); // Print failed tests if any if (summary.failedTests.length > 0) { @@ -125,7 +125,7 @@ function parseResults() { // Print unimplemented tests summary (grouped by major version) if (summary.unimplementedTests.length > 0) { - console.log('\n=== UNIMPLEMENTED TESTS (Informational) ==='); + console.log('\n=== OPTIONAL FEATURES NOT IMPLEMENTED (Informational) ==='); // Group by major test category const unimplementedByCategory = {}; From e22f5bb042655c4c065f9ba43da32c6e4d81b322 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 5 Oct 2025 23:27:44 +0000 Subject: [PATCH 7/7] Exclude non-strict and informational tests from pass rate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Non-strict and informational tests are not failures, they represent acceptable behavior variations, so they should not count against the pass rate. Changes: - Pass rate now excludes: optional + non-strict + informational tests - Simplified label from 'Pass rate (required)' to 'Pass rate' - Pass rate now shows actual pass/fail ratio for strict tests Example with current implementation: - Before: 97.7% (294 / 301) - After: 100.0% (294 / 294) Calculation: OK / (Total - Optional - Non-Strict - Informational) = 294 / (517 - 216 - 4 - 3) = 294 / 294 = 100% This accurately reflects that all strict required tests pass with zero failures. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- test/autobahn/parse-results.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/autobahn/parse-results.js b/test/autobahn/parse-results.js index dc1c53ff..ce0628d9 100755 --- a/test/autobahn/parse-results.js +++ b/test/autobahn/parse-results.js @@ -93,11 +93,12 @@ function parseResults() { console.log(` Non-Strict: ${summary.nonStrict}`); console.log(` Informational: ${summary.informational}`); - const requiredTests = summary.total - summary.unimplemented; - const passRate = requiredTests > 0 - ? ((summary.ok / requiredTests) * 100).toFixed(1) + // Pass rate excludes optional, non-strict, and informational tests + const strictRequired = summary.total - summary.unimplemented - summary.nonStrict - summary.informational; + const passRate = strictRequired > 0 + ? ((summary.ok / strictRequired) * 100).toFixed(1) : '0.0'; - console.log(` Pass rate (required): ${passRate}%`); + console.log(` Pass rate: ${passRate}%`); // Print failed tests if any if (summary.failedTests.length > 0) {