diff --git a/.github/workflows/websocket-tests.yml b/.github/workflows/websocket-tests.yml index f98d3956..fd022cae 100644 --- a/.github/workflows/websocket-tests.yml +++ b/.github/workflows/websocket-tests.yml @@ -20,3 +20,9 @@ jobs: - run: pnpm run lint - run: pnpm run test + + - name: Pull Autobahn Test Suite Docker image + run: docker pull crossbario/autobahn-testsuite + + - name: Run Autobahn WebSocket Protocol Tests + run: cd test/autobahn && node run-wstest.js diff --git a/test/autobahn/config/fuzzingclient-linux.json b/test/autobahn/config/fuzzingclient-linux.json new file mode 100644 index 00000000..0859770c --- /dev/null +++ b/test/autobahn/config/fuzzingclient-linux.json @@ -0,0 +1,17 @@ + +{ + "options": {"failByDrop": false}, + "outdir": "./reports/servers", + + "servers": [ + { + "agent": "WebSocket-Node 1.0.27", + "url": "ws://localhost:8080", + "options": {"version": 18} + } + ], + + "cases": ["*"], + "exclude-cases": [], + "exclude-agent-cases": {} +} diff --git a/test/autobahn/parse-results.js b/test/autobahn/parse-results.js index 63850b52..ce0628d9 100755 --- a/test/autobahn/parse-results.js +++ b/test/autobahn/parse-results.js @@ -86,13 +86,18 @@ function parseResults() { // Print summary console.log('Test Summary:'); console.log(` Total tests: ${summary.total}`); + console.log(` Required tests: ${summary.total - summary.unimplemented}`); + console.log(` Optional tests: ${summary.unimplemented}`); console.log(` Passed (OK): ${summary.ok}`); console.log(` Failed: ${summary.failed}`); console.log(` Non-Strict: ${summary.nonStrict}`); console.log(` Informational: ${summary.informational}`); - console.log(` Unimplemented: ${summary.unimplemented}`); - - const passRate = ((summary.ok / summary.total) * 100).toFixed(1); + + // Pass rate excludes optional, non-strict, and informational tests + const strictRequired = summary.total - summary.unimplemented - summary.nonStrict - summary.informational; + const passRate = strictRequired > 0 + ? ((summary.ok / strictRequired) * 100).toFixed(1) + : '0.0'; console.log(` Pass rate: ${passRate}%`); // Print failed tests if any @@ -121,7 +126,7 @@ function parseResults() { // Print unimplemented tests summary (grouped by major version) if (summary.unimplementedTests.length > 0) { - console.log('\n=== UNIMPLEMENTED TESTS (Informational) ==='); + console.log('\n=== OPTIONAL FEATURES NOT IMPLEMENTED (Informational) ==='); // Group by major test category const unimplementedByCategory = {}; @@ -140,7 +145,7 @@ function parseResults() { } console.log('\n'); - + // Exit with error code if there are actual failures if (summary.failed > 0) { console.error(`❌ ${summary.failed} test(s) failed!`); @@ -148,6 +153,8 @@ function parseResults() { } else { console.log(`✅ All tests passed! (${summary.ok} OK, ${summary.nonStrict} non-strict, ${summary.informational} informational, ${summary.unimplemented} unimplemented)`); } + + return summary; } if (require.main === module) { diff --git a/test/autobahn/run-wstest.js b/test/autobahn/run-wstest.js index 765434a1..621c6659 100755 --- a/test/autobahn/run-wstest.js +++ b/test/autobahn/run-wstest.js @@ -106,16 +106,22 @@ class AutobahnTestRunner { runAutobahnTests() { return new Promise((resolve, reject) => { console.log('🐳 Starting Autobahn test suite with Docker...'); - + + // Detect platform and use appropriate config and networking + const isLinux = process.platform === 'linux'; + const configFile = isLinux ? 'fuzzingclient-linux.json' : 'fuzzingclient.json'; + + console.log(` Platform: ${process.platform}, using config: ${configFile}`); + const dockerArgs = [ 'run', '--rm', + ...(isLinux ? ['--network=host'] : ['-p', '9001:9001']), '-v', `${process.cwd()}/config:/config`, '-v', `${process.cwd()}/reports:/reports`, - '-p', '9001:9001', '--name', 'fuzzingclient', 'crossbario/autobahn-testsuite', - 'wstest', '-m', 'fuzzingclient', '--spec', '/config/fuzzingclient.json' + 'wstest', '-m', 'fuzzingclient', '--spec', `/config/${configFile}` ]; this.dockerProcess = spawn('docker', dockerArgs, { @@ -157,26 +163,36 @@ class AutobahnTestRunner { parseAndDisplayResults() { console.log('📊 Parsing test results...\n'); - + const resultsPath = path.join(__dirname, 'reports', 'servers', 'index.json'); - + if (!fs.existsSync(resultsPath)) { console.error('❌ Results file not found. Tests may not have completed properly.'); - return; + process.exit(1); } try { const originalProcessExit = process.exit; - // Prevent parseResults from exiting the process - process.exit = () => {}; - - parseResults(); - + let exitCode = 0; + + // Intercept process.exit to capture the exit code + process.exit = (code) => { + exitCode = code || 0; + }; + + const summary = parseResults(); + // Restore original function process.exit = originalProcessExit; - + + // Exit with appropriate code if there were failures + if (exitCode !== 0 || (summary && summary.failed > 0)) { + process.exit(1); + } + } catch (error) { console.error('❌ Failed to parse results:', error.message); + process.exit(1); } }