From e59105927b687b1312b0692633d014b187efe519 Mon Sep 17 00:00:00 2001 From: Paolo Insogna Date: Thu, 7 May 2026 07:34:28 +0200 Subject: [PATCH] ffi: add experimental fast FFI call API Signed-off-by: Paolo Insogna Assisted-By: OpenAI:GPT-5.5 --- benchmark/ffi/{add-f64.js => add-64.js} | 2 +- benchmark/ffi/add-f32.js | 28 ++ benchmark/ffi/add-i16.js | 28 ++ benchmark/ffi/add-i64.js | 28 ++ benchmark/ffi/add-i8.js | 28 ++ benchmark/ffi/add-u16.js | 28 ++ benchmark/ffi/add-u64.js | 28 ++ benchmark/ffi/add-u8.js | 28 ++ benchmark/ffi/buffer-first-byte-direct.js | 30 ++ benchmark/ffi/buffer-first-byte.js | 29 ++ .../{sum-buffer.js => buffer-sum-direct.js} | 16 +- benchmark/ffi/buffer-sum.js | 30 ++ benchmark/ffi/identity-i32.js | 28 ++ benchmark/ffi/noop-void.js | 28 ++ benchmark/ffi/pointer-buffer-direct.js | 30 ++ benchmark/ffi/pointer-buffer.js | 29 ++ benchmark/ffi/pointer-null.js | 28 ++ .../ffi/string-equals-hello-buffer-direct.js | 30 ++ benchmark/ffi/string-equals-hello-buffer.js | 29 ++ .../ffi/string-first-char-buffer-direct.js | 30 ++ benchmark/ffi/string-first-char-buffer.js | 29 ++ benchmark/ffi/string-length-buffer-direct.js | 30 ++ benchmark/ffi/string-length-buffer.js | 29 ++ benchmark/ffi/string-length-string-direct.js | 30 ++ benchmark/ffi/string-length-string.js | 28 ++ benchmark/ffi/sum-3-i32.js | 28 ++ benchmark/ffi/sum-5-i32.js | 28 ++ benchmark/ffi/sum-8-i32.js | 28 ++ doc/api/ffi.md | 8 + doc/contributing/ffi-fast-api-internals.md | 314 ++++++++++++++ lib/ffi.js | 114 ++++- lib/internal/ffi-shared-buffer.js | 92 ---- lib/internal/ffi/fast-api.js | 344 +++++++++++++++ node.gyp | 3 + src/env_properties.h | 2 + src/ffi/fast.cc | 295 +++++++++++++ src/ffi/fast.h | 73 ++++ src/ffi/platforms/arm64.cc | 395 ++++++++++++++++++ src/node_ffi.cc | 114 ++++- src/node_ffi.h | 3 + test/ffi/fixture_library/ffi_test_library.c | 40 ++ test/ffi/test-ffi-calls.js | 41 ++ test/ffi/test-ffi-fast-buffer.js | 71 ++++ test/ffi/test-ffi-shared-buffer.js | 50 ++- 44 files changed, 2595 insertions(+), 129 deletions(-) rename benchmark/ffi/{add-f64.js => add-64.js} (96%) create mode 100644 benchmark/ffi/add-f32.js create mode 100644 benchmark/ffi/add-i16.js create mode 100644 benchmark/ffi/add-i64.js create mode 100644 benchmark/ffi/add-i8.js create mode 100644 benchmark/ffi/add-u16.js create mode 100644 benchmark/ffi/add-u64.js create mode 100644 benchmark/ffi/add-u8.js create mode 100644 benchmark/ffi/buffer-first-byte-direct.js create mode 100644 benchmark/ffi/buffer-first-byte.js rename benchmark/ffi/{sum-buffer.js => buffer-sum-direct.js} (67%) create mode 100644 benchmark/ffi/buffer-sum.js create mode 100644 benchmark/ffi/identity-i32.js create mode 100644 benchmark/ffi/noop-void.js create mode 100644 benchmark/ffi/pointer-buffer-direct.js create mode 100644 benchmark/ffi/pointer-buffer.js create mode 100644 benchmark/ffi/pointer-null.js create mode 100644 benchmark/ffi/string-equals-hello-buffer-direct.js create mode 100644 benchmark/ffi/string-equals-hello-buffer.js create mode 100644 benchmark/ffi/string-first-char-buffer-direct.js create mode 100644 benchmark/ffi/string-first-char-buffer.js create mode 100644 benchmark/ffi/string-length-buffer-direct.js create mode 100644 benchmark/ffi/string-length-buffer.js create mode 100644 benchmark/ffi/string-length-string-direct.js create mode 100644 benchmark/ffi/string-length-string.js create mode 100644 benchmark/ffi/sum-3-i32.js create mode 100644 benchmark/ffi/sum-5-i32.js create mode 100644 benchmark/ffi/sum-8-i32.js create mode 100644 doc/contributing/ffi-fast-api-internals.md create mode 100644 lib/internal/ffi/fast-api.js create mode 100644 src/ffi/fast.cc create mode 100644 src/ffi/fast.h create mode 100644 src/ffi/platforms/arm64.cc create mode 100644 test/ffi/test-ffi-fast-buffer.js diff --git a/benchmark/ffi/add-f64.js b/benchmark/ffi/add-64.js similarity index 96% rename from benchmark/ffi/add-f64.js rename to benchmark/ffi/add-64.js index fab6457e33a09d..e398cb8d236654 100644 --- a/benchmark/ffi/add-f64.js +++ b/benchmark/ffi/add-64.js @@ -21,7 +21,7 @@ const add = functions.add_f64; function main({ n }) { bench.start(); for (let i = 0; i < n; ++i) - add(1.5, 2.5); + add(20.5, 21.5); bench.end(n); lib.close(); diff --git a/benchmark/ffi/add-f32.js b/benchmark/ffi/add-f32.js new file mode 100644 index 00000000000000..0d1072543d0aa3 --- /dev/null +++ b/benchmark/ffi/add-f32.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + add_f32: { result: 'f32', parameters: ['f32', 'f32'] }, +}); + +const add = functions.add_f32; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + add(20.5, 21.5); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/add-i16.js b/benchmark/ffi/add-i16.js new file mode 100644 index 00000000000000..5a3abb730b2281 --- /dev/null +++ b/benchmark/ffi/add-i16.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + add_i16: { result: 'i16', parameters: ['i16', 'i16'] }, +}); + +const add = functions.add_i16; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + add(20, 22); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/add-i64.js b/benchmark/ffi/add-i64.js new file mode 100644 index 00000000000000..e8cac89bfae543 --- /dev/null +++ b/benchmark/ffi/add-i64.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + add_i64: { result: 'i64', parameters: ['i64', 'i64'] }, +}); + +const add = functions.add_i64; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + add(20n, 22n); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/add-i8.js b/benchmark/ffi/add-i8.js new file mode 100644 index 00000000000000..d0c3041faee154 --- /dev/null +++ b/benchmark/ffi/add-i8.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + add_i8: { result: 'i8', parameters: ['i8', 'i8'] }, +}); + +const add = functions.add_i8; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + add(20, 22); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/add-u16.js b/benchmark/ffi/add-u16.js new file mode 100644 index 00000000000000..7d98511cd05691 --- /dev/null +++ b/benchmark/ffi/add-u16.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + add_u16: { result: 'u16', parameters: ['u16', 'u16'] }, +}); + +const add = functions.add_u16; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + add(20, 22); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/add-u64.js b/benchmark/ffi/add-u64.js new file mode 100644 index 00000000000000..f30a0fe72557fa --- /dev/null +++ b/benchmark/ffi/add-u64.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + add_u64: { result: 'u64', parameters: ['u64', 'u64'] }, +}); + +const add = functions.add_u64; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + add(20n, 22n); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/add-u8.js b/benchmark/ffi/add-u8.js new file mode 100644 index 00000000000000..254c3085abf412 --- /dev/null +++ b/benchmark/ffi/add-u8.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + add_u8: { result: 'u8', parameters: ['u8', 'u8'] }, +}); + +const add = functions.add_u8; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + add(20, 22); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/buffer-first-byte-direct.js b/benchmark/ffi/buffer-first-byte-direct.js new file mode 100644 index 00000000000000..aba5d3c873761b --- /dev/null +++ b/benchmark/ffi/buffer-first-byte-direct.js @@ -0,0 +1,30 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + first_byte: { result: 'u8', parameters: ['pointer'] }, +}); + +const fn = functions.first_byte; +const bytes = Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]); +const pointer = ffi.getRawPointer(bytes); + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(pointer); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/buffer-first-byte.js b/benchmark/ffi/buffer-first-byte.js new file mode 100644 index 00000000000000..5ca8d8ce515d07 --- /dev/null +++ b/benchmark/ffi/buffer-first-byte.js @@ -0,0 +1,29 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + first_byte: { result: 'u8', parameters: ['buffer'] }, +}); + +const fn = functions.first_byte; +const bytes = Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]); + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(bytes); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/sum-buffer.js b/benchmark/ffi/buffer-sum-direct.js similarity index 67% rename from benchmark/ffi/sum-buffer.js rename to benchmark/ffi/buffer-sum-direct.js index 3117f61aaedabf..f923069f392b48 100644 --- a/benchmark/ffi/sum-buffer.js +++ b/benchmark/ffi/buffer-sum-direct.js @@ -5,8 +5,7 @@ const ffi = require('node:ffi'); const { libraryPath, ensureFixtureLibrary } = require('./common.js'); const bench = common.createBenchmark(main, { - size: [64, 1024, 16384], - n: [1e6], + n: [1e7], }, { flags: ['--experimental-ffi'], }); @@ -17,16 +16,15 @@ const { lib, functions } = ffi.dlopen(libraryPath, { sum_buffer: { result: 'u64', parameters: ['pointer', 'u64'] }, }); -function main({ n, size }) { - const buf = Buffer.alloc(size, 0x42); - const ptr = ffi.getRawPointer(buf); - const len = BigInt(size); - - const sum = functions.sum_buffer; +const fn = functions.sum_buffer; +const bytes = Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]); +const pointer = ffi.getRawPointer(bytes); +const length = BigInt(bytes.length); +function main({ n }) { bench.start(); for (let i = 0; i < n; ++i) - sum(ptr, len); + fn(pointer, length); bench.end(n); lib.close(); diff --git a/benchmark/ffi/buffer-sum.js b/benchmark/ffi/buffer-sum.js new file mode 100644 index 00000000000000..8a2d49880afed6 --- /dev/null +++ b/benchmark/ffi/buffer-sum.js @@ -0,0 +1,30 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + sum_buffer: { result: 'u64', parameters: ['buffer', 'u64'] }, +}); + +const fn = functions.sum_buffer; +const bytes = Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]); +const length = BigInt(bytes.length); + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(bytes, length); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/identity-i32.js b/benchmark/ffi/identity-i32.js new file mode 100644 index 00000000000000..f05f5f18ebc1dd --- /dev/null +++ b/benchmark/ffi/identity-i32.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + identity_i32: { result: 'i32', parameters: ['i32'] }, +}); + +const fn = functions.identity_i32; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(42); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/noop-void.js b/benchmark/ffi/noop-void.js new file mode 100644 index 00000000000000..d11427ddd894df --- /dev/null +++ b/benchmark/ffi/noop-void.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + noop_void: { result: 'void', parameters: [] }, +}); + +const fn = functions.noop_void; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/pointer-buffer-direct.js b/benchmark/ffi/pointer-buffer-direct.js new file mode 100644 index 00000000000000..be769d949bf8a4 --- /dev/null +++ b/benchmark/ffi/pointer-buffer-direct.js @@ -0,0 +1,30 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + pointer_to_usize: { result: 'u64', parameters: ['pointer'] }, +}); + +const fn = functions.pointer_to_usize; +const bytes = Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]); +const pointer = ffi.getRawPointer(bytes); + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(pointer); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/pointer-buffer.js b/benchmark/ffi/pointer-buffer.js new file mode 100644 index 00000000000000..e9066ade78706d --- /dev/null +++ b/benchmark/ffi/pointer-buffer.js @@ -0,0 +1,29 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + pointer_to_usize: { result: 'u64', parameters: ['pointer'] }, +}); + +const fn = functions.pointer_to_usize; +const bytes = Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]); + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(bytes); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/pointer-null.js b/benchmark/ffi/pointer-null.js new file mode 100644 index 00000000000000..f4fd8930b02b7b --- /dev/null +++ b/benchmark/ffi/pointer-null.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + is_null_pointer: { result: 'u8', parameters: ['pointer'] }, +}); + +const fn = functions.is_null_pointer; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(null); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/string-equals-hello-buffer-direct.js b/benchmark/ffi/string-equals-hello-buffer-direct.js new file mode 100644 index 00000000000000..5768ca6c376799 --- /dev/null +++ b/benchmark/ffi/string-equals-hello-buffer-direct.js @@ -0,0 +1,30 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + string_equals_hello: { result: 'u8', parameters: ['pointer'] }, +}); + +const fn = functions.string_equals_hello; +const string = Buffer.from('hello\0'); +const pointer = ffi.getRawPointer(string); + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(pointer); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/string-equals-hello-buffer.js b/benchmark/ffi/string-equals-hello-buffer.js new file mode 100644 index 00000000000000..0c89e2eb3a1c94 --- /dev/null +++ b/benchmark/ffi/string-equals-hello-buffer.js @@ -0,0 +1,29 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + string_equals_hello: { result: 'u8', parameters: ['buffer'] }, +}); + +const fn = functions.string_equals_hello; +const string = Buffer.from('hello\0'); + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(string); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/string-first-char-buffer-direct.js b/benchmark/ffi/string-first-char-buffer-direct.js new file mode 100644 index 00000000000000..79db4cae0d8739 --- /dev/null +++ b/benchmark/ffi/string-first-char-buffer-direct.js @@ -0,0 +1,30 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + string_first_char: { result: 'u8', parameters: ['pointer'] }, +}); + +const fn = functions.string_first_char; +const string = Buffer.from('hello\0'); +const pointer = ffi.getRawPointer(string); + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(pointer); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/string-first-char-buffer.js b/benchmark/ffi/string-first-char-buffer.js new file mode 100644 index 00000000000000..408b3c15a15e26 --- /dev/null +++ b/benchmark/ffi/string-first-char-buffer.js @@ -0,0 +1,29 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + string_first_char: { result: 'u8', parameters: ['buffer'] }, +}); + +const fn = functions.string_first_char; +const string = Buffer.from('hello\0'); + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(string); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/string-length-buffer-direct.js b/benchmark/ffi/string-length-buffer-direct.js new file mode 100644 index 00000000000000..99e47bf9570dcb --- /dev/null +++ b/benchmark/ffi/string-length-buffer-direct.js @@ -0,0 +1,30 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + string_length: { result: 'u64', parameters: ['pointer'] }, +}); + +const fn = functions.string_length; +const string = Buffer.from('hello\0'); +const pointer = ffi.getRawPointer(string); + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(pointer); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/string-length-buffer.js b/benchmark/ffi/string-length-buffer.js new file mode 100644 index 00000000000000..d9a061c3d505ac --- /dev/null +++ b/benchmark/ffi/string-length-buffer.js @@ -0,0 +1,29 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + string_length: { result: 'u64', parameters: ['buffer'] }, +}); + +const fn = functions.string_length; +const string = Buffer.from('hello\0'); + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(string); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/string-length-string-direct.js b/benchmark/ffi/string-length-string-direct.js new file mode 100644 index 00000000000000..99e47bf9570dcb --- /dev/null +++ b/benchmark/ffi/string-length-string-direct.js @@ -0,0 +1,30 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + string_length: { result: 'u64', parameters: ['pointer'] }, +}); + +const fn = functions.string_length; +const string = Buffer.from('hello\0'); +const pointer = ffi.getRawPointer(string); + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(pointer); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/string-length-string.js b/benchmark/ffi/string-length-string.js new file mode 100644 index 00000000000000..0f5284da8630c5 --- /dev/null +++ b/benchmark/ffi/string-length-string.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + string_length: { result: 'u64', parameters: ['string'] }, +}); + +const fn = functions.string_length; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn('hello'); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/sum-3-i32.js b/benchmark/ffi/sum-3-i32.js new file mode 100644 index 00000000000000..3ba162e49c1543 --- /dev/null +++ b/benchmark/ffi/sum-3-i32.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + sum_3_i32: { result: 'i32', parameters: ['i32', 'i32', 'i32'] }, +}); + +const fn = functions.sum_3_i32; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(10, 11, 21); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/sum-5-i32.js b/benchmark/ffi/sum-5-i32.js new file mode 100644 index 00000000000000..3559b4b3b2fd14 --- /dev/null +++ b/benchmark/ffi/sum-5-i32.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + sum_5_i32: { result: 'i32', parameters: ['i32', 'i32', 'i32', 'i32', 'i32'] }, +}); + +const fn = functions.sum_5_i32; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(2, 4, 8, 12, 16); + bench.end(n); + + lib.close(); +} diff --git a/benchmark/ffi/sum-8-i32.js b/benchmark/ffi/sum-8-i32.js new file mode 100644 index 00000000000000..1d12e76d85d0d2 --- /dev/null +++ b/benchmark/ffi/sum-8-i32.js @@ -0,0 +1,28 @@ +'use strict'; + +const common = require('../common.js'); +const ffi = require('node:ffi'); +const { libraryPath, ensureFixtureLibrary } = require('./common.js'); + +const bench = common.createBenchmark(main, { + n: [1e7], +}, { + flags: ['--experimental-ffi'], +}); + +ensureFixtureLibrary(); + +const { lib, functions } = ffi.dlopen(libraryPath, { + sum_8_i32: { result: 'i32', parameters: ['i32', 'i32', 'i32', 'i32', 'i32', 'i32', 'i32', 'i32'] }, +}); + +const fn = functions.sum_8_i32; + +function main({ n }) { + bench.start(); + for (let i = 0; i < n; ++i) + fn(1, 2, 3, 4, 5, 6, 7, 14); + bench.end(n); + + lib.close(); +} diff --git a/doc/api/ffi.md b/doc/api/ffi.md index 930ad70afc0155..e7955b631c3ccd 100644 --- a/doc/api/ffi.md +++ b/doc/api/ffi.md @@ -118,6 +118,14 @@ is signed it behaves like `i8`; otherwise it behaves like `u8`. The `bool` type is marshaled as an 8-bit unsigned integer. Pass numeric values such as `0` and `1`; JavaScript `true` and `false` are not accepted. +On optimized Fast FFI calls, `pointer`, `ptr`, and `function` parameters accept +raw pointer `bigint` values. For pointer-like parameters, `null`, `undefined`, +strings, `Buffer`, typed array, `DataView`, and `ArrayBuffer` values are +converted on the JavaScript side before calling the optimized native wrapper. + +Optimized Fast FFI calls support at most 8 function arguments. Functions with +more than 7 arguments use the generic FFI call path instead. + ## Signature objects Functions and callbacks are described with signature objects. diff --git a/doc/contributing/ffi-fast-api-internals.md b/doc/contributing/ffi-fast-api-internals.md new file mode 100644 index 00000000000000..7a72ed2c91c331 --- /dev/null +++ b/doc/contributing/ffi-fast-api-internals.md @@ -0,0 +1,314 @@ +# FFI Fast API internals + +This document describes the internal implementation of the `node:ffi` Fast API +path. It is intended for contributors working on the FFI implementation, not for +users of the public API. + +The Fast API path is an optimization layer for FFI calls whose signatures can be +represented by V8 Fast API metadata and a generated native trampoline. It does +not replace the generic libffi path. Instead, Node.js creates the fastest +available callable for each signature and keeps the generic path available for +unsupported call shapes, deoptimized V8 calls, and validation behavior that must +match the public FFI API. + +## Goals + +The Fast API implementation is designed around these goals: + +* Keep hot scalar FFI calls out of the generic `v8::FunctionCallbackInfo` path. +* Avoid per-call allocation for common numeric and pointer-like signatures. +* Preserve the public `node:ffi` behavior and error shape. +* Keep string lifetime management in JavaScript, where temporary buffers can be + owned explicitly. +* Keep SharedBuffer and Fast API routing separate, with `lib/ffi.js` composing + the two wrapper layers. +* Use generated per-signature native code instead of runtime loops inside the + trampoline. + +## Main files + +The implementation is split across these files: + +* `src/ffi/fast.h` declares the Fast API type model, metadata containers, and + platform trampoline hooks. +* `src/ffi/fast.cc` maps public FFI type names to Fast API types, creates V8 + `CFunctionInfo` metadata, and exposes buffer conversion helpers used by + generated trampolines. +* `src/ffi/platforms/*.cc` contains the platform trampoline generators. These + files follow the contract exposed by `node_ffi_create_fast_trampoline()`. +* `src/node_ffi.cc` decides whether a function gets a Fast API callable, + SharedBuffer callable, or generic callable, and attaches hidden metadata used + by JavaScript wrappers. +* `src/node_ffi.h` stores `FastFFIMetadata` objects in `FFIFunctionInfo` so V8 + metadata and generated executable code stay alive with the JavaScript function. +* `lib/ffi.js` is the public module wrapper. It patches `DynamicLibrary` methods + and composes SharedBuffer and Fast API wrappers. +* `lib/internal/ffi/fast-api.js` performs JavaScript-side pointer conversions + for Fast API calls. +* `lib/internal/ffi-shared-buffer.js` contains only SharedBuffer-specific + argument packing and result unpacking. + +## Native metadata creation + +`DynamicLibrary::CreateFunction()` creates one `FFIFunctionInfo` per generated +JavaScript function. That object owns the parsed `FFIFunction`, the persistent +function handle, the owning library reference, and any optimized invocation +metadata. + +The creation flow is: + +1. Call `CreateFastFFIMetadata(*fn)`. +2. If Fast API metadata is created, bind the JavaScript function with a V8 + `FunctionTemplate` that contains both the conventional callback and the Fast + API `v8::CFunction`. +3. If Fast API metadata cannot be created, try the SharedBuffer path for + eligible signatures. +4. If neither optimized path applies, bind the generic `InvokeFunction` path. + +Returning `nullptr` from `CreateFastFFIMetadata()` is not a public signature +error. It means only that the current Fast API implementation cannot optimize +that signature. The caller then falls back to another invocation strategy. + +## FastFFIType + +The internal `FastFFIType` enum is intentionally smaller than the public FFI type +surface. It models the ABI categories that the generated trampoline knows how to +marshal directly: + +* `kVoid` +* `kBool` +* signed and unsigned 8-bit, 16-bit, 32-bit, and 64-bit integers +* `kFloat32` +* `kFloat64` +* `kPointer` +* `kBuffer` + +Public aliases are normalized in `FastScalarTypeFromName()` and +`FastArgTypeFromName()`. + +`pointer`, `ptr`, `function`, `string`, `str`, `buffer`, and `arraybuffer` all +represent pointer-sized native values at the target ABI boundary. They differ in +how JavaScript values are accepted and converted before the target function is +called. + +## V8 CFunction metadata + +`CreateFastFFIMetadata()` creates a `FastFFIMetadata` object. That object owns: + +* `FastFFITrampoline trampoline`, the executable bridge called by V8. +* `std::vector arg_info`, the V8 type list. +* `std::unique_ptr c_function_info`, the V8 signature object. +* `v8::CFunction c_function`, the handle attached to the `FunctionTemplate`. + +The metadata object must own `arg_info` and `c_function_info` because V8 keeps +pointers into that storage. Destroying or moving that storage while the function +is alive would leave V8 with dangling pointers. + +The first V8 argument is always the JavaScript receiver. For that reason, +`CreateFastFFIMetadata()` prepends a `v8::CTypeInfo::Type::kV8Value` entry before +the public FFI arguments. + +If any argument or return value needs a 64-bit integer or pointer, the V8 +`CFunctionInfo` is configured with BigInt representation. This avoids precision +loss for pointer-sized values and 64-bit integers. + +## Generated trampolines + +The generated trampoline bridges V8's Fast API calling convention to the native +ABI expected by the library symbol. Its responsibilities are: + +* Move incoming V8 Fast API arguments into the registers expected by the target + native function. +* Narrow 8-bit and 16-bit integer arguments after V8 has widened them. +* Convert `kBuffer` arguments from V8 values to backing-store pointers. +* Call the target symbol. +* Normalize narrow integer return values before V8 observes them. +* Return control to V8 using the platform ABI. + +The trampoline is generated per signature. It does not loop over arguments at +runtime using metadata tables. The code generator may loop while emitting +instructions, but the emitted code is a straight-line bridge specialized for the +signature. + +Executable memory is allocated writable, populated with instructions, flushed +from the instruction cache as required by the platform, and then marked +executable. `FastFFIMetadata` releases that memory through +`node_ffi_free_fast_trampoline()` when the JavaScript function is collected. + +## Buffer and ArrayBuffer arguments + +Fast API buffer arguments are represented internally as `FastFFIType::kBuffer`. +In V8 metadata they are described as `v8::Local`, not as `uint64_t`. +This lets the generated trampoline receive the original JavaScript object and +call `node_ffi_fast_buffer_data()` immediately before the native target call. + +`node_ffi_fast_buffer_data()` accepts: + +* `Buffer` and other ArrayBuffer views +* `ArrayBuffer` +* `SharedArrayBuffer` + +For views, the pointer is the backing store plus `byteOffset`. For ArrayBuffer +and SharedArrayBuffer, the pointer is the start of the backing store. + +Invalid values cause the helper to throw through `FastApiCallbackOptions` and +return a sentinel value. The generated trampoline checks for that sentinel and +returns zero without calling the native target. This prevents native code from +observing an invalid pointer after JavaScript validation has failed. + +## Pointer-like arguments + +Pointer-like public types include `pointer`, `ptr`, and `function`. They are +represented as raw unsigned pointer values in the scalar Fast API signature. + +JavaScript wrappers in `lib/internal/ffi/fast-api.js` convert accepted non-BigInt +values before entering the scalar Fast API function: + +* `null` and `undefined` become `0n`. +* strings become temporary NUL-terminated UTF-8 buffers. +* Buffer and ArrayBuffer-backed values become backing-store pointers. + +Keeping these conversions in JavaScript preserves public FFI semantics and keeps +temporary object lifetime explicit. + +## String arguments + +String conversion intentionally stays in JavaScript. A string accepted by a +`string`, `str`, or pointer-like parameter is encoded into a temporary Buffer +with a trailing NUL byte. The pointer to that Buffer is passed to the scalar Fast +API function. + +Each owning `DynamicLibrary` keeps a hidden array of string conversion entries. +Each argument index gets one reusable entry. If the same string is passed again +at the same argument index, the wrapper reuses the existing encoded buffer and +pointer. This is a single-entry reuse strategy, not an unbounded cache. + +This design avoids native lifetime ambiguity. The generated trampoline never +allocates temporary string storage and never has to guess how long a pointer to a +converted string must stay alive. + +## Secondary buffer invoke for pointer signatures + +A single pointer-like function can be called efficiently in two different ways: + +* with a scalar pointer value, such as `BigInt` or `null` +* with a memory object, such as `Buffer` or `ArrayBuffer` + +These two cases require different V8 Fast API representations. The scalar case +uses a `uint64_t`/BigInt-shaped argument. The memory-object case uses a +`v8::Local` argument so the generated trampoline can extract the +backing-store pointer. + +For monomorphic single-argument pointer-like signatures, native code may attach +a secondary function under the hidden `kFastBufferInvoke` symbol. This secondary +function uses a cloned signature where the pointer-like argument is described as +`buffer` for Fast API metadata purposes, while still calling the same native +target symbol. + +The JavaScript wrapper dispatches to this secondary function only when the +runtime argument is Buffer or ArrayBuffer-backed memory. Other pointer inputs use +the primary scalar Fast API function. + +This keeps both important call shapes fast. Replacing the primary scalar Fast +API function with the buffer-shaped function would simplify the machinery, but +it would force BigInt, null, and string-converted pointer calls onto a slower +fallback path. Keeping both entrypoints preserves performance for both pointer +representations. + +## Hidden symbols + +Native code attaches internal metadata to raw generated functions using +per-isolate Symbols. These symbols are exported through `internalBinding('ffi')` +and are not part of the public API. + +SharedBuffer metadata uses: + +* `kSbSharedBuffer` +* `kSbInvokeSlow` +* `kSbParams` +* `kSbResult` + +Fast API metadata uses: + +* `kFastParams` +* `kFastBufferInvoke` + +The two groups are intentionally separate. SharedBuffer wrappers should not need +Fast API metadata, and Fast API wrappers should not need SharedBuffer metadata. +`lib/ffi.js` is the composition layer that reads both groups and decides which +wrapper to apply. + +## JavaScript wrapper routing + +`lib/ffi.js` patches the native `DynamicLibrary` methods that expose generated +functions: + +* `DynamicLibrary.prototype.getFunction` +* `DynamicLibrary.prototype.getFunctions` +* the `DynamicLibrary.prototype.functions` accessor + +All three routes call `wrapFFIFunction()` before returning functions to user +code. + +`wrapFFIFunction()` applies wrappers in this order: + +1. Initialize Fast API buffer metadata for raw Fast API functions. +2. Try `wrapWithSharedBuffer()`. +3. If SharedBuffer did not apply, try `wrapWithRawPointerConversions()`. +4. Return the original raw function unchanged if no wrapper is needed. + +This ordering keeps SharedBuffer-specific behavior inside +`lib/internal/ffi-shared-buffer.js`, Fast API pointer conversion behavior inside +`lib/internal/ffi/fast-api.js`, and public wrapper orchestration inside +`lib/ffi.js`. + +## SharedBuffer fallback + +SharedBuffer remains a separate optimized path for signatures that are not using +Fast API but can still avoid per-argument native conversion overhead. The +SharedBuffer wrapper writes arguments into a fixed 8-byte slot layout, calls a +raw native function with no JavaScript arguments, and reads the return value from +slot zero. + +Pointer signatures using SharedBuffer have a slow invoker attached under +`kSbInvokeSlow`. The wrapper uses the SharedBuffer path for BigInt and nullish +pointer values, and falls back to the slow invoker for values that require the +generic native conversion path. + +Fast API and SharedBuffer are independent. A function uses either the Fast API +path, the SharedBuffer path, or the generic path as its primary native callable. +`lib/ffi.js` only composes the JavaScript wrappers needed to preserve public +argument behavior. + +## Generic fallback behavior + +Every Fast API function is also bound with the conventional native callback. +V8 can call that callback when a JavaScript call site is not eligible for the +Fast API path. Node.js also uses the generic path directly when metadata creation +rejects a signature. + +The generic path remains responsible for complete validation and public error +compatibility. Fast wrappers should match those errors for conversions they +perform in JavaScript. + +## Argument count and signature limits + +The Fast API path intentionally supports only a bounded number of function +arguments. This keeps V8 metadata, wrapper specialization, and generated +trampolines simple and predictable. Signatures outside that bound fall back to +SharedBuffer or the generic path. + +This is an optimization boundary, not a public FFI signature boundary. User code +can still call supported public FFI signatures through the fallback paths. + +## Wrapper metadata preservation + +JavaScript wrappers preserve selected public function metadata: + +* `name` +* `length` +* `pointer` + +The `pointer` property mirrors the raw function's pointer descriptor so user +code that reads or reassigns it continues to work through wrappers. Internal +Symbol-keyed metadata is not forwarded to wrappers. \ No newline at end of file diff --git a/lib/ffi.js b/lib/ffi.js index 98af095e0cb01c..c494b860a16708 100644 --- a/lib/ffi.js +++ b/lib/ffi.js @@ -1,7 +1,11 @@ 'use strict'; const { + FunctionPrototypeCall, + ObjectDefineProperty, ObjectFreeze, + ObjectGetOwnPropertyDescriptor, + ObjectKeys, ObjectPrototypeToString, SymbolDispose, } = primordials; @@ -13,6 +17,7 @@ const { const { codes: { ERR_ACCESS_DENIED, + ERR_INTERNAL_ASSERTION, ERR_INVALID_ARG_TYPE, ERR_OUT_OF_RANGE, }, @@ -39,6 +44,8 @@ const { getFloat64, exportBytes, getRawPointer, + kFastParams, + kSbParams, setInt8, setUint8, setInt16, @@ -54,12 +61,117 @@ const { toArrayBuffer, } = internalBinding('ffi'); -require('internal/ffi-shared-buffer'); +const { + wrapWithSharedBuffer, +} = require('internal/ffi-shared-buffer'); + +const { + initializeFastBufferMetadata, + wrapWithRawPointerConversions, +} = require('internal/ffi/fast-api'); DynamicLibrary.prototype[SymbolDispose] = function() { this.close(); }; +// Accept-set mirrors the native `ParseFunctionSignature` in +// `src/ffi/types.cc`. `ParseFunctionSignature` additionally throws when +// multiple aliases are set at once. The wrapper runs before the native +// call, so those conflicts still surface from the native side regardless +// of which alias we happen to read here. +function sigParams(sig) { + return sig.parameters ?? sig.arguments ?? []; +} + +function sigResult(sig) { + return sig.result ?? sig.return ?? sig.returns ?? 'void'; +} + +function wrapFFIFunction(rawFn, parameters, resultType, owner) { + if (parameters === undefined && rawFn !== undefined && rawFn !== null) { + parameters = rawFn[kSbParams] ?? rawFn[kFastParams]; + } + initializeFastBufferMetadata(rawFn, parameters); + const wrapped = wrapWithSharedBuffer(rawFn, parameters, resultType); + if (wrapped !== rawFn) { + return wrapped; + } + return wrapWithRawPointerConversions(rawFn, parameters, owner); +} + +// Every public path that surfaces native-created FFI functions goes through the +// wrapper dispatcher here. The SharedBuffer and Fast API modules only expose +// their own wrapping primitives; this file decides which one applies. +const rawGetFunction = DynamicLibrary.prototype.getFunction; +const rawGetFunctions = DynamicLibrary.prototype.getFunctions; + +DynamicLibrary.prototype.getFunction = function getFunction(name, sig) { + // Native `DynamicLibrary::GetFunction` validates `sig`, so by the time + // we have `raw` we know `sig` is a valid object. + const raw = FunctionPrototypeCall(rawGetFunction, this, name, sig); + return wrapFFIFunction(raw, sigParams(sig), sigResult(sig), this); +}; + +DynamicLibrary.prototype.getFunctions = function getFunctions(definitions) { + // Native `GetFunctions` switches on `args.Length() > 0`. Zero args + // returns every cached function, one arg requires an object. Forwarding + // `undefined` would fail the object check, so drop it when omitted. + const raw = definitions === undefined ? + FunctionPrototypeCall(rawGetFunctions, this) : + FunctionPrototypeCall(rawGetFunctions, this, definitions); + if (raw === undefined || raw === null) { + return raw; + } + const keys = ObjectKeys(raw); + const out = { __proto__: null }; + for (let i = 0; i < keys.length; i++) { + const name = keys[i]; + // No `definitions`: native side returned every cached function, so use + // each raw function's hidden metadata to decide whether it needs wrapping. + if (definitions === undefined) { + out[name] = wrapFFIFunction(raw[name], undefined, undefined, this); + } else { + const sig = definitions[name]; + out[name] = wrapFFIFunction( + raw[name], sigParams(sig), sigResult(sig), this); + } + } + return out; +}; + +{ + // The native side installs `functions` as an accessor returning raw cached + // functions. Rewrap each access so both SharedBuffer and Fast API pointer + // conversion wrappers are applied consistently. + const functionsDescriptor = + ObjectGetOwnPropertyDescriptor(DynamicLibrary.prototype, 'functions'); + /* c8 ignore start */ + if (functionsDescriptor === undefined || !functionsDescriptor.get) { + throw new ERR_INTERNAL_ASSERTION( + 'FFI: DynamicLibrary.prototype.functions accessor not found or has no getter'); + } + /* c8 ignore stop */ + const origGetter = functionsDescriptor.get; + ObjectDefineProperty(DynamicLibrary.prototype, 'functions', { + __proto__: null, + configurable: true, + enumerable: functionsDescriptor.enumerable, + get() { + const raw = FunctionPrototypeCall(origGetter, this); + if (raw === undefined || raw === null) { + return raw; + } + const wrapped = { __proto__: null }; + const keys = ObjectKeys(raw); + for (let i = 0; i < keys.length; i++) { + const name = keys[i]; + wrapped[name] = wrapFFIFunction(raw[name], undefined, undefined, this); + } + return wrapped; + }, + }); +} + function checkFFIPermission() { if (!permission.isEnabled() || permission.has('ffi')) { return; diff --git a/lib/internal/ffi-shared-buffer.js b/lib/internal/ffi-shared-buffer.js index bce51fd79959dd..e005cc8c770909 100644 --- a/lib/internal/ffi-shared-buffer.js +++ b/lib/internal/ffi-shared-buffer.js @@ -22,11 +22,8 @@ const { DataViewPrototypeSetUint16, DataViewPrototypeSetUint32, DataViewPrototypeSetUint8, - FunctionPrototypeCall, NumberIsInteger, ObjectDefineProperty, - ObjectGetOwnPropertyDescriptor, - ObjectKeys, ReflectApply, TypeError, } = primordials; @@ -38,7 +35,6 @@ const { } = require('internal/errors'); const { - DynamicLibrary, charIsSigned, kSbInvokeSlow, kSbParams, @@ -542,94 +538,6 @@ function buildNumericWrapper( }; } -// Accept-set mirrors the native `ParseFunctionSignature` in -// `src/ffi/types.cc`. `ParseFunctionSignature` additionally throws when -// multiple aliases are set at once. The wrapper runs before the native -// call, so those conflicts still surface from the native side regardless -// of which alias we happen to read here. -function sigParams(sig) { - return sig.parameters ?? sig.arguments ?? []; -} - -function sigResult(sig) { - return sig.result ?? sig.return ?? sig.returns ?? 'void'; -} - -// The native invoker for SB-eligible symbols is `InvokeFunctionSB`, which -// reads arguments from the shared buffer populated by -// `wrapWithSharedBuffer`. These patches make sure every path that surfaces -// a raw SB-eligible function to user code (`getFunction`, `getFunctions`, -// and the `functions` accessor) returns the wrapper instead. -const rawGetFunction = DynamicLibrary.prototype.getFunction; -const rawGetFunctions = DynamicLibrary.prototype.getFunctions; - -DynamicLibrary.prototype.getFunction = function getFunction(name, sig) { - // Native `DynamicLibrary::GetFunction` validates `sig`, so by the time - // we have `raw` we know `sig` is a valid object. - const raw = FunctionPrototypeCall(rawGetFunction, this, name, sig); - return wrapWithSharedBuffer(raw, sigParams(sig), sigResult(sig)); -}; - -DynamicLibrary.prototype.getFunctions = function getFunctions(definitions) { - // Native `GetFunctions` switches on `args.Length() > 0`. Zero args - // returns every cached function, one arg requires an object. Forwarding - // `undefined` would fail the object check, so drop it when omitted. - const raw = definitions === undefined ? - FunctionPrototypeCall(rawGetFunctions, this) : - FunctionPrototypeCall(rawGetFunctions, this, definitions); - if (raw === undefined || raw === null) return raw; - const keys = ObjectKeys(raw); - const out = { __proto__: null }; - for (let i = 0; i < keys.length; i++) { - const name = keys[i]; - // No `definitions`: native side returned every cached function, so we - // wrap using each function's own `kSbParams` / `kSbResult` metadata - // (same fallback as the `functions` accessor). - if (definitions === undefined) { - out[name] = wrapWithSharedBuffer(raw[name]); - } else { - const sig = definitions[name]; - out[name] = wrapWithSharedBuffer(raw[name], sigParams(sig), sigResult(sig)); - } - } - return out; -}; - -{ - // The native side installs `functions` as an accessor returning raw - // functions. Rewrap each access so `lib.functions.foo(...)` goes through - // the SB wrapper instead of invoking the fast path against an - // uninitialized buffer. - const functionsDescriptor = - ObjectGetOwnPropertyDescriptor(DynamicLibrary.prototype, 'functions'); - /* c8 ignore start */ - if (functionsDescriptor === undefined || !functionsDescriptor.get) { - // Missing getter means the native and JS sides are out of sync; silently - // skipping the patch would expose the fast-path-against-uninitialized-buffer - // footgun this whole block exists to prevent. - throw new ERR_INTERNAL_ASSERTION( - 'FFI: DynamicLibrary.prototype.functions accessor not found or has no getter'); - } - /* c8 ignore stop */ - const origGetter = functionsDescriptor.get; - ObjectDefineProperty(DynamicLibrary.prototype, 'functions', { - __proto__: null, - configurable: true, - enumerable: functionsDescriptor.enumerable, - get() { - const raw = FunctionPrototypeCall(origGetter, this); - if (raw === undefined || raw === null) return raw; - const wrapped = { __proto__: null }; - const keys = ObjectKeys(raw); - for (let i = 0; i < keys.length; i++) { - const name = keys[i]; - wrapped[name] = wrapWithSharedBuffer(raw[name]); - } - return wrapped; - }, - }); -} - module.exports = { wrapWithSharedBuffer, }; diff --git a/lib/internal/ffi/fast-api.js b/lib/internal/ffi/fast-api.js new file mode 100644 index 00000000000000..03b97b698918ce --- /dev/null +++ b/lib/internal/ffi/fast-api.js @@ -0,0 +1,344 @@ +'use strict'; + +const { + ArrayPrototypeIncludes, + ObjectDefineProperty, + ReflectApply, + StringPrototypeIncludes, + Symbol, + TypeError, +} = primordials; + +const { + Buffer, +} = require('buffer'); + +const { + isAnyArrayBuffer, + isArrayBufferView, +} = require('internal/util/types'); + +const { + getRawPointer, + kFastBufferInvoke, + kFastParams, + kSbSharedBuffer, +} = internalBinding('ffi'); + +// kFastBuffer is JS-local metadata. Native code only exposes whether a raw +// function has Fast API parameter metadata; this marker records that this +// wrapper has already recognized a native buffer-shaped fast path. +const kFastBuffer = Symbol('kFastBuffer'); + +// String conversion buffers are stored on the owning DynamicLibrary so their +// backing stores stay alive for the duration of the FFI call and can be reused +// across repeated calls to the same signature slot. +const kStringConversionBuffer = Symbol('kStringConversionBuffer'); + +function throwFFIArgError(msg) { + // eslint-disable-next-line no-restricted-syntax + const err = new TypeError(msg); + err.code = 'ERR_INVALID_ARG_VALUE'; + throw err; +} + +function throwFFIArgCountError(expected, actual) { + throwFFIArgError( + `Invalid argument count: expected ${expected}, got ${actual}`); +} + +function needsRawPointerConversion(type, rawFn) { + // Native Fast API buffer parameters already receive the original V8 value and + // convert it inside the generated trampoline. Do not pre-convert those to a + // BigInt pointer in JS or V8 would miss the buffer-shaped fast signature. + if (rawFn !== undefined && rawFn[kFastBuffer] === true && + (type === 'buffer' || type === 'arraybuffer')) { + return false; + } + return type === 'buffer' || type === 'arraybuffer'; +} + +function needsPointerLikeConversion(type) { + // These public aliases are ABI-identical: each ultimately becomes an unsigned + // pointer-sized integer for the scalar Fast API call. + return type === 'pointer' || type === 'ptr' || type === 'function'; +} + +function needsStringPointerConversion(type) { + // Plain string signatures and pointer-like signatures both accept JS strings. + // The temporary NUL-terminated UTF-8 buffer must be owned by JS, not by the + // generated native trampoline. + return type === 'string' || type === 'str' || needsPointerLikeConversion(type); +} + +function needsNullPointerConversion(type) { + // Nullish values are accepted for all pointer-shaped parameters and are + // normalized to the native null pointer before entering the Fast API call. + return needsPointerLikeConversion(type) || type === 'string' || type === 'str' || + needsRawPointerConversion(type); +} + +function needsPointerConversion(type, rawFn) { + // This is the broad wrapper predicate. It intentionally covers string, + // nullish, Buffer/ArrayBuffer, and raw object-to-pointer conversion cases. + if (rawFn !== undefined && rawFn[kFastBuffer] === true && + (type === 'buffer' || type === 'arraybuffer')) { + return false; + } + return needsRawPointerConversion(type, rawFn) || + needsNullPointerConversion(type) || needsStringPointerConversion(type); +} + +function hasStringPointerArg(type, value) { + // SharedBuffer uses this helper too, so keep the runtime value check separate + // from the type-level predicate. + return typeof value === 'string' && needsStringPointerConversion(type); +} + +function hasPointerMemoryArg(type, value) { + // Buffer and ArrayBuffer-backed values can be converted to a backing-store + // address for raw pointer-like signatures. + return (needsRawPointerConversion(type) || needsStringPointerConversion(type)) && + (isArrayBufferView(value) || isAnyArrayBuffer(value)); +} + +function getStringConversionPointer(owner, value, index) { + // Allocate pessimistically for UTF-8 (`3 * length`) plus the trailing NUL. + // The exact byte count is known only after encoding. + const size = value.length * 3 + 1; + let buffers = owner[kStringConversionBuffer]; + if (buffers === undefined) { + // Keep one conversion slot per argument index. This avoids per-call Symbol + // lookups on the wrapper function and keeps buffers tied to the library + // lifetime rather than a temporary wrapper invocation. + buffers = []; + ObjectDefineProperty(owner, kStringConversionBuffer, { + __proto__: null, + configurable: false, + enumerable: false, + writable: false, + value: buffers, + }); + } + let entry = buffers[index]; + if (entry !== undefined && entry.string === value) { + // Single-entry same-string reuse handles hot loops such as strlen("foo") + // without introducing an unbounded string-to-buffer cache. + return entry.pointer; + } + if (StringPrototypeIncludes(value, '\0')) { + throwFFIArgError(`Argument ${index} must not contain null bytes`); + } + if (entry === undefined || entry.buffer.length < size) { + // Grow-only reuse: keep the existing backing store when it is large enough, + // otherwise allocate a bigger Buffer and cache its raw pointer once. + const buffer = Buffer.allocUnsafe(size); + entry = { + __proto__: null, + buffer, + pointer: getRawPointer(buffer), + string: undefined, + }; + buffers[index] = entry; + } + + const buffer = entry.buffer; + // Encode directly into the reusable Buffer and append the C terminator that + // native string/pointer consumers expect. + const written = buffer.write(value, 0, size - 1, 'utf8'); + buffer[written] = 0; + entry.string = value; + return entry.pointer; +} + +function convertPointerArg(type, value, owner, index) { + // Preserve the conversion order used by public FFI semantics: nullish first, + // then strings with temporary ownership, then memory-backed objects, then the + // generic raw pointer extraction path for buffer/arraybuffer declarations. + if (needsNullPointerConversion(type) && + (value === null || value === undefined)) { + return 0n; + } + if (hasStringPointerArg(type, value)) { + return getStringConversionPointer(owner, value, index); + } + if (hasPointerMemoryArg(type, value)) { + return getRawPointer(value); + } + if (needsRawPointerConversion(type)) { + return getRawPointer(value); + } + return value; +} + +function getPointerConversionIndexes(parameters, rawFn) { + // Return null instead of [] so callers can cheaply detect that no wrapper is + // needed and leave the native Fast API function untouched. + let indexes = null; + for (let i = 0; i < parameters.length; i++) { + if (!needsPointerConversion(parameters[i], rawFn)) { + continue; + } + if (indexes === null) { + indexes = []; + } + indexes.push(i); + } + return indexes; +} + +function initializeFastBufferMetadata(rawFn, parameters) { + // This function annotates raw Fast API functions after `lib/ffi.js` has the + // user-facing signature. It deliberately skips SharedBuffer functions because + // those are routed by `wrapWithSharedBuffer()` instead. + if (rawFn === undefined || rawFn === null || parameters === undefined) { + return; + } + if (rawFn[kSbSharedBuffer] !== undefined) { + return; + } + + const nativeParams = rawFn[kFastParams]; + if (nativeParams !== undefined) { + // If native metadata exists and the user signature contains buffer-shaped + // parameters, keep them as V8 values so the generated trampoline can use + // node_ffi_fast_buffer_data(). + for (let i = 0; i < parameters.length; i++) { + const type = parameters[i]; + if (type === 'buffer' || type === 'arraybuffer') { + rawFn[kFastBuffer] = true; + break; + } + } + } + + // The native side attaches `kFastBufferInvoke` for monomorphic pointer-like + // signatures when a secondary buffer trampoline exists. +} + +// The `pointer` descriptor mirrors the raw function's so user code that +// reassigns `.pointer` keeps working through the wrapper. +function inheritMetadata(wrapper, rawFn, nargs) { + ObjectDefineProperty(wrapper, 'name', { + __proto__: null, value: rawFn.name, configurable: true, + }); + ObjectDefineProperty(wrapper, 'length', { + __proto__: null, value: nargs, configurable: true, + }); + ObjectDefineProperty(wrapper, 'pointer', { + __proto__: null, value: rawFn.pointer, + writable: true, configurable: true, enumerable: true, + }); + return wrapper; +} + +function wrapWithRawPointerConversions(rawFn, parameters, owner) { + // The raw function is returned unchanged when there is no Fast API metadata or + // no argument requires JS-side pointer conversion. + if (rawFn === undefined || rawFn === null) { + return rawFn; + } + if (parameters === undefined) { + parameters = rawFn[kFastParams]; + } + if (parameters === undefined) { + return rawFn; + } + + const indexes = getPointerConversionIndexes(parameters, rawFn); + if (indexes === null) { + return rawFn; + } + + const nargs = parameters.length; + let wrapper; + if (nargs === 1 && indexes.length === 1 && indexes[0] === 0) { + // The monomorphic one-argument wrapper is the hot path for pointer/string + // helpers. It avoids allocating a rest-args array and can dispatch directly + // to the secondary buffer-shaped Fast API function when available. + const t0 = parameters[0]; + const string0 = needsStringPointerConversion(t0); + const memory0 = needsRawPointerConversion(t0) || string0; + const fastBufferInvoke = needsPointerLikeConversion(t0) ? + rawFn[kFastBufferInvoke] : undefined; + wrapper = function(a0) { + if (arguments.length !== 1) { + throwFFIArgCountError(1, arguments.length); + } + let arg = a0; + if (needsNullPointerConversion(t0) && + (arg === null || arg === undefined)) { + arg = 0n; + } else if (string0 && typeof arg === 'string') { + arg = getStringConversionPointer(owner, arg, 0); + } else if (memory0 && (isArrayBufferView(arg) || isAnyArrayBuffer(arg))) { + if (fastBufferInvoke !== undefined) { + // Keep two Fast API representations for single pointer-like + // signatures: scalar u64 for BigInt/null/string-converted pointers, + // and buffer-shaped for Buffer/ArrayBuffer inputs. Replacing one with + // the other would make one of those cases fall back to a slower path. + return fastBufferInvoke(arg); + } + arg = getRawPointer(arg); + } + const result = rawFn(arg); + return result; + }; + } else if (nargs === 2) { + // Small fixed-arity wrappers preserve the raw function's call shape and + // avoid rest-args allocation while still converting only the needed slots. + const c0 = ArrayPrototypeIncludes(indexes, 0); + const c1 = ArrayPrototypeIncludes(indexes, 1); + const t0 = parameters[0]; + const t1 = parameters[1]; + wrapper = function(a0, a1) { + if (arguments.length !== 2) { + throwFFIArgCountError(2, arguments.length); + } + return rawFn(c0 ? convertPointerArg(t0, a0, owner, 0) : a0, + c1 ? convertPointerArg(t1, a1, owner, 1) : a1); + }; + } else if (nargs === 3) { + // Three arguments is the last fixed specialization currently worth keeping; + // larger signatures use the generic loop below to avoid code bloat. + const c0 = ArrayPrototypeIncludes(indexes, 0); + const c1 = ArrayPrototypeIncludes(indexes, 1); + const c2 = ArrayPrototypeIncludes(indexes, 2); + const t0 = parameters[0]; + const t1 = parameters[1]; + const t2 = parameters[2]; + wrapper = function(a0, a1, a2) { + if (arguments.length !== 3) { + throwFFIArgCountError(3, arguments.length); + } + return rawFn(c0 ? convertPointerArg(t0, a0, owner, 0) : a0, + c1 ? convertPointerArg(t1, a1, owner, 1) : a1, + c2 ? convertPointerArg(t2, a2, owner, 2) : a2); + }; + } else { + // Generic fallback for larger signatures. Mutating the rest-args array is + // safe because it is invocation-local, then ReflectApply forwards the final + // converted argument list to the raw Fast API function. + wrapper = function(...args) { + if (args.length !== nargs) { + throwFFIArgCountError(nargs, args.length); + } + for (let i = 0; i < indexes.length; i++) { + const index = indexes[i]; + args[index] = convertPointerArg( + parameters[index], args[index], owner, index); + } + return ReflectApply(rawFn, undefined, args); + }; + } + + return inheritMetadata(wrapper, rawFn, nargs); +} + +module.exports = { + convertPointerArg, + hasPointerMemoryArg, + hasStringPointerArg, + initializeFastBufferMetadata, + wrapWithRawPointerConversions, +}; diff --git a/node.gyp b/node.gyp index b129c3db8d88c1..9b7999804cd555 100644 --- a/node.gyp +++ b/node.gyp @@ -469,8 +469,11 @@ 'node_ffi_sources': [ 'src/node_ffi.cc', 'src/node_ffi.h', + 'src/ffi/platforms/arm64.cc', 'src/ffi/data.cc', 'src/ffi/data.h', + 'src/ffi/fast.cc', + 'src/ffi/fast.h', 'src/ffi/types.cc', 'src/ffi/types.h', ], diff --git a/src/env_properties.h b/src/env_properties.h index 0fc7b2b66179e4..427eb3ba5d4194 100644 --- a/src/env_properties.h +++ b/src/env_properties.h @@ -50,6 +50,8 @@ V(ffi_sb_invoke_slow_symbol, "ffi_sb_invoke_slow_symbol") \ V(ffi_sb_params_symbol, "ffi_sb_params_symbol") \ V(ffi_sb_result_symbol, "ffi_sb_result_symbol") \ + V(ffi_fast_params_symbol, "ffi_fast_params_symbol") \ + V(ffi_fast_buffer_invoke_symbol, "ffi_fast_buffer_invoke_symbol") \ V(constructor_key_symbol, "constructor_key_symbol") \ V(handle_onclose_symbol, "handle_onclose") \ V(no_message_symbol, "no_message_symbol") \ diff --git a/src/ffi/fast.cc b/src/ffi/fast.cc new file mode 100644 index 00000000000000..b411abae877def --- /dev/null +++ b/src/ffi/fast.cc @@ -0,0 +1,295 @@ +#if HAVE_FFI + +#include "ffi/fast.h" + +#include "env-inl.h" +#include "node_errors.h" +#include "node_ffi.h" + +#include +#include + +namespace node::ffi { + +using v8::CFunctionInfo; +using v8::CTypeInfo; +using v8::FastApiCallbackOptions; + +bool IsTypeName(std::string_view type, + std::initializer_list names) { + // Signature parsing accepts several public aliases for the same ABI type. + // The fast path normalizes them by checking the original type name against + // each alias set before selecting a FastFFIType. + for (std::string_view name : names) { + if (type == name) { + return true; + } + } + return false; +} + +bool FastScalarTypeFromName(std::string_view type, FastFFIType* out) { + // Scalars can be passed directly through V8 Fast API CFunction metadata. + // Pointer-like types are represented as uintptr-sized unsigned integers; + // JavaScript wrappers handle strings and object-to-pointer conversions. + if (type == "void") { + *out = FastFFIType::kVoid; + } else if (type == "bool") { + *out = FastFFIType::kBool; + } else if (IsTypeName(type, {"i8", "int8"})) { + *out = FastFFIType::kInt8; + } else if (IsTypeName(type, {"u8", "uint8"})) { + *out = FastFFIType::kUint8; + } else if (type == "char") { + *out = CHAR_MIN < 0 ? FastFFIType::kInt8 : FastFFIType::kUint8; + } else if (IsTypeName(type, {"i16", "int16"})) { + *out = FastFFIType::kInt16; + } else if (IsTypeName(type, {"u16", "uint16"})) { + *out = FastFFIType::kUint16; + } else if (IsTypeName(type, {"i32", "int32"})) { + *out = FastFFIType::kInt32; + } else if (IsTypeName(type, {"u32", "uint32"})) { + *out = FastFFIType::kUint32; + } else if (IsTypeName(type, {"i64", "int64"})) { + *out = FastFFIType::kInt64; + } else if (IsTypeName(type, {"u64", "uint64"})) { + *out = FastFFIType::kUint64; + } else if (IsTypeName(type, {"f32", "float", "float32"})) { + *out = FastFFIType::kFloat32; + } else if (IsTypeName(type, {"f64", "double", "float64"})) { + *out = FastFFIType::kFloat64; + } else if (IsTypeName(type, {"buffer", "arraybuffer"})) { + *out = FastFFIType::kPointer; + } else if (IsTypeName(type, + {"pointer", "ptr", "string", "str", "function"})) { + *out = FastFFIType::kPointer; + } else { + return false; + } + return true; +} + +bool FastArgTypeFromName(std::string_view type, FastFFIType* out) { + // Buffer and ArrayBuffer parameters are special only for arguments: V8 passes + // the original JS value so the generated trampoline can extract its backing + // store address immediately before calling the native target. + if (IsTypeName(type, {"buffer", "arraybuffer"})) { + *out = FastFFIType::kBuffer; + return true; + } + return FastScalarTypeFromName(type, out); +} + +bool NeedsBigIntRepresentation(FastFFIType type) { + // V8's Fast API has one int64 representation per CFunction. Any pointer or + // 64-bit integer in the signature forces BigInt to avoid precision loss. + return type == FastFFIType::kInt64 || type == FastFFIType::kUint64 || + type == FastFFIType::kPointer; +} + +CTypeInfo::Type ToV8Type(FastFFIType type, bool is_return) { + // Map the internal FFI type model onto the limited set of CFunctionInfo + // types that V8 can call directly. Narrow integer arguments are widened in + // the V8 signature and then narrowed again by the trampoline. + switch (type) { + case FastFFIType::kVoid: + return CTypeInfo::Type::kVoid; + case FastFFIType::kBool: + return CTypeInfo::Type::kBool; + case FastFFIType::kUint8: + return CTypeInfo::Type::kUint32; + case FastFFIType::kInt8: + case FastFFIType::kInt16: + return CTypeInfo::Type::kInt32; + case FastFFIType::kUint16: + return CTypeInfo::Type::kUint32; + case FastFFIType::kInt32: + return CTypeInfo::Type::kInt32; + case FastFFIType::kUint32: + return CTypeInfo::Type::kUint32; + case FastFFIType::kInt64: + return CTypeInfo::Type::kInt64; + case FastFFIType::kUint64: + case FastFFIType::kPointer: + return CTypeInfo::Type::kUint64; + case FastFFIType::kBuffer: + return CTypeInfo::Type::kV8Value; + case FastFFIType::kFloat32: + return CTypeInfo::Type::kFloat32; + case FastFFIType::kFloat64: + return CTypeInfo::Type::kFloat64; + } + UNREACHABLE(); +} + +uintptr_t PointerFromValue(v8::Local value) { + // ArrayBufferViews point at the backing store plus their byte offset, while + // ArrayBuffer and SharedArrayBuffer point at the start of their store. + if (value->IsArrayBufferView()) { + v8::Local view = value.As(); + void* data = view->Buffer()->Data(); + return reinterpret_cast(static_cast(data) + + view->ByteOffset()); + } + + if (value->IsArrayBuffer()) { + return reinterpret_cast(value.As()->Data()); + } + + if (value->IsSharedArrayBuffer()) { + std::shared_ptr store = + value.As()->GetBackingStore(); + return reinterpret_cast(store->Data()); + } + + return std::numeric_limits::max(); +} + +bool SignatureNeedsRawPointerConversions(const FFIFunction& fn) { + // These public types accept JS values that are not raw BigInt pointers. When + // a signature contains them, JS wraps the fast function to perform the + // conversion before V8 enters the CFunction trampoline. + for (const std::string& name : fn.arg_type_names) { + if (name == "buffer" || name == "arraybuffer" || name == "string" || + name == "str") { + return true; + } + } + return false; +} + +bool IsPointerTypeName(const std::string& name) { + // `pointer`, `ptr`, and `function` all use the same uintptr ABI slot; only + // the public type spelling differs. + return name == "pointer" || name == "ptr" || name == "function"; +} + +bool SignatureNeedsFastBufferInvoke(const FFIFunction& fn) { + // The secondary buffer invoke is only generated for the hot monomorphic case + // where a single pointer-like argument can be satisfied by a Buffer or + // ArrayBuffer without allocating or caching a BigInt pointer in JS. + return fn.arg_type_names.size() == 1 && + IsPointerTypeName(fn.arg_type_names[0]); +} + +std::shared_ptr CloneWithFastBufferArgNames( + const std::shared_ptr& fn) { + // Reuse the same native target and libffi metadata, but describe the JS + // argument as `buffer` so CreateFastFFIMetadata() emits a trampoline that + // receives a V8 value and calls node_ffi_fast_buffer_data(). + auto clone = std::make_shared(*fn); + for (std::string& name : clone->arg_type_names) { + if (IsPointerTypeName(name)) { + name = "buffer"; + } + } + return clone; +} + +extern "C" uintptr_t node_ffi_fast_buffer_data( + v8::Local value, + FastApiCallbackOptions* options, + uint32_t index) { + // The generated trampoline treats this sentinel as "conversion failed" and + // returns zero after throwing, preventing the native target from seeing an + // invalid pointer value. + constexpr uintptr_t kInvalidBuffer = std::numeric_limits::max(); + + // Accept only memory-backed JS values in the native helper. Other pointer + // conversions, including strings, stay in the JS wrapper so their temporary + // lifetime is explicit. + if (value->IsArrayBufferView()) { + return PointerFromValue(value); + } + if (value->IsArrayBuffer() || value->IsSharedArrayBuffer()) { + return PointerFromValue(value); + } + + v8::Isolate* isolate = options != nullptr ? options->isolate : nullptr; + if (isolate != nullptr) { + THROW_ERR_INVALID_ARG_VALUE( + isolate, "Argument %u must be a buffer or an ArrayBuffer", index); + } + return kInvalidBuffer; +} + +FastFFIMetadata::~FastFFIMetadata() { + // Metadata owns executable memory through `trampoline`; releasing it here + // ties code lifetime to the V8 function's weak FFIFunctionInfo cleanup. + node_ffi_free_fast_trampoline(&trampoline); +} + +std::unique_ptr CreateFastFFIMetadata(const FFIFunction& fn) { + // Reject unsupported result types first. Returning nullptr means the caller + // can still fall back to SharedBuffer or the generic libffi path. + FastFFIType result; + if (!FastScalarTypeFromName(fn.return_type_name, &result)) { + return nullptr; + } + if (fn.args.size() != fn.arg_type_names.size()) { + return nullptr; + } + // Keep the initial Fast API implementation bounded to signatures V8 and the + // platform trampolines can describe without stack argument support. + if (fn.arg_type_names.size() > 8) { + return nullptr; + } + + std::vector args; + args.reserve(fn.arg_type_names.size()); + bool needs_bigint = NeedsBigIntRepresentation(result); + bool needs_callback_options = false; + // Normalize public argument names into FastFFIType values while collecting + // signature-wide flags required by V8 CFunctionInfo. + for (const std::string& name : fn.arg_type_names) { + FastFFIType type; + if (!FastArgTypeFromName(name, &type)) { + return nullptr; + } + if (type == FastFFIType::kVoid) { + return nullptr; + } + needs_bigint = needs_bigint || NeedsBigIntRepresentation(type); + needs_callback_options = needs_callback_options || + type == FastFFIType::kBuffer; + args.push_back(type); + } + + auto metadata = std::make_unique(); + // The platform-specific trampoline is the executable entrypoint V8 calls. + // If the platform rejects the signature, the whole fast metadata object is + // discarded and the caller chooses another invocation path. + if (!node_ffi_create_fast_trampoline( + fn.ptr, args.data(), args.size(), result, &metadata->trampoline)) { + return nullptr; + } + + metadata->arg_info.reserve(args.size() + 1); + // Fast API callbacks include the JS receiver as an implicit first argument, + // so the generated CFunctionInfo begins with a V8Value placeholder for it. + metadata->arg_info.emplace_back(CTypeInfo::Type::kV8Value); + for (FastFFIType arg : args) { + metadata->arg_info.emplace_back(ToV8Type(arg, false)); + } + // Buffer arguments require FastApiCallbackOptions so the helper can throw on + // invalid inputs and report the correct argument index. + if (needs_callback_options) { + metadata->arg_info.emplace_back(CTypeInfo::kCallbackOptionsType); + } + + // CFunctionInfo stores pointers into `arg_info`; both are held by + // FastFFIMetadata so they remain valid for the generated function lifetime. + metadata->c_function_info = std::make_unique( + CTypeInfo(ToV8Type(result, true)), + metadata->arg_info.size(), + metadata->arg_info.data(), + needs_bigint ? CFunctionInfo::Int64Representation::kBigInt + : CFunctionInfo::Int64Representation::kNumber); + metadata->c_function = v8::CFunction(metadata->trampoline.code, + metadata->c_function_info.get()); + return metadata; +} + +} // namespace node::ffi + +#endif // HAVE_FFI diff --git a/src/ffi/fast.h b/src/ffi/fast.h new file mode 100644 index 00000000000000..c050afe8e4e5b1 --- /dev/null +++ b/src/ffi/fast.h @@ -0,0 +1,73 @@ +#pragma once + +#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#include "ffi.h" +#include "v8-fast-api-calls.h" + +#include +#include +#include +#include +#include + +namespace node::ffi { + +struct FFIFunction; + +enum class FastFFIType : uint8_t { + kVoid, + kBool, + kInt8, + kUint8, + kInt16, + kUint16, + kInt32, + kUint32, + kInt64, + kUint64, + kFloat32, + kFloat64, + kPointer, + kBuffer, +}; + +struct FastFFITrampoline { + void* code = nullptr; + size_t size = 0; +}; + +struct FastFFIMetadata { + FastFFIMetadata() = default; + ~FastFFIMetadata(); + + FastFFIMetadata(const FastFFIMetadata&) = delete; + FastFFIMetadata& operator=(const FastFFIMetadata&) = delete; + + FastFFITrampoline trampoline; + std::vector arg_info; + std::unique_ptr c_function_info; + v8::CFunction c_function; +}; + +bool SignatureNeedsRawPointerConversions(const FFIFunction& fn); +bool IsPointerTypeName(const std::string& name); +bool SignatureNeedsFastBufferInvoke(const FFIFunction& fn); +std::shared_ptr CloneWithFastBufferArgNames(const std::shared_ptr& fn); +std::unique_ptr CreateFastFFIMetadata(const FFIFunction& fn); + +} // namespace node::ffi + +extern "C" { +uintptr_t node_ffi_fast_buffer_data(v8::Local value, + v8::FastApiCallbackOptions* options, + uint32_t index); +bool node_ffi_create_fast_trampoline(void* target, + const node::ffi::FastFFIType* args, + size_t argc, + node::ffi::FastFFIType result, + node::ffi::FastFFITrampoline* out); +void node_ffi_free_fast_trampoline(node::ffi::FastFFITrampoline* trampoline); +} + +#endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS diff --git a/src/ffi/platforms/arm64.cc b/src/ffi/platforms/arm64.cc new file mode 100644 index 00000000000000..43f918d49f30ab --- /dev/null +++ b/src/ffi/platforms/arm64.cc @@ -0,0 +1,395 @@ +#if HAVE_FFI + +#include "ffi/fast.h" + +#if defined(__aarch64__) || defined(_M_ARM64) + +#include +#include + +#include + +#include + +#if defined(__APPLE__) +#include +#endif + +namespace { + +using node::ffi::FastFFIType; + +// Fixed instruction encodings used verbatim in every trampoline. These are the +// prologue/epilogue and indirect-call instructions that do not depend on a +// register number or immediate value. +constexpr uint32_t kBlrX16 = 0xd63f0200; +constexpr uint32_t kStpFpLrPreIndex = 0xa9bf7bfd; +constexpr uint32_t kLdpFpLrPostIndex = 0xa8c17bfd; +constexpr uint32_t kRet = 0xd65f03c0; + +// Floating-point values use vN registers in the AArch64 ABI. Everything else +// uses general-purpose xN registers and follows the GP shuffle path below. +bool IsFloatType(FastFFIType type) { + return type == FastFFIType::kFloat32 || type == FastFFIType::kFloat64; +} + +// The trampoline only needs to distinguish FP from non-FP ABI classes for the +// current scalar implementation. +bool IsGPType(FastFFIType type) { + return !IsFloatType(type); +} + +// kBuffer means V8 passes the original JS value; the trampoline must call back +// into C++ to convert it to a native pointer before invoking the target. +bool IsPointerValueType(FastFFIType type) { + return type == FastFFIType::kBuffer; +} + +// The helpers below encode the small set of AArch64 instructions needed by the +// generated trampoline. Keeping them as functions makes the emitted sequence +// readable without pulling in a full assembler dependency. +uint32_t MovX(unsigned dst, unsigned src) { + // mov Xdst, Xsrc is encoded as ORR with xzr. It is used for register + // shuffles between V8's calling convention and the target ABI. + return 0xaa0003e0 | (src << 16) | dst; +} + +uint32_t SubSp(unsigned imm) { + // Reserve stack spill space with `sub sp, sp, #imm`. The caller guarantees + // 16-byte alignment and keeps the immediate within the 12-bit encoding. + return 0xd10003ff | ((imm & 0xfff) << 10); +} + +uint32_t AddSp(unsigned imm) { + // Release stack spill space with `add sp, sp, #imm`, matching SubSp(). + return 0x910003ff | ((imm & 0xfff) << 10); +} + +uint32_t StrXSp(unsigned reg, unsigned offset) { + // Store a 64-bit GP register at [sp + offset]. Offsets are slot-based in the + // instruction encoding, so byte offsets are divided by 8. + return 0xf90003e0 | ((offset / 8) << 10) | reg; +} + +uint32_t LdrXSp(unsigned reg, unsigned offset) { + // Reload a 64-bit GP register from [sp + offset], using the same spill-slot + // layout as StrXSp(). + return 0xf94003e0 | ((offset / 8) << 10) | reg; +} + +uint32_t MovzW(unsigned dst, uint16_t value) { + // Load a small immediate into a W register. The buffer helper's argument + // index is uint32_t, but current Fast API signatures are capped well below + // the 16-bit immediate range. + return 0x52800000 | (value << 5) | dst; +} + +uint32_t CmpX(unsigned lhs, unsigned rhs) { + // Compare two X registers by encoding SUBS xzr, lhs, rhs. Used to test the + // helper result against the invalid-pointer sentinel loaded into x16. + return 0xeb00001f | (rhs << 16) | (lhs << 5); +} + +uint32_t BCond(unsigned instruction_offset, unsigned cond) { + // Emit a conditional branch with an instruction-count offset. The buffer + // helper path uses cond=1 (`ne`) to skip the early-return sequence when the + // converted pointer is valid. + return 0x54000000 | ((instruction_offset & 0x7ffff) << 5) | cond; +} + +uint32_t MovzX16(uint16_t value, unsigned shift) { + // Start materializing a 64-bit absolute address in x16, 16 bits at a time. + return 0xd2800000 | ((shift / 16) << 21) | (value << 5) | 16; +} + +uint32_t MovkX16(uint16_t value, unsigned shift) { + // Patch the next 16-bit lane of x16 without disturbing the lanes already + // written by MovzX16()/previous MovkX16() instructions. + return 0xf2800000 | ((shift / 16) << 21) | (value << 5) | 16; +} + +uint32_t SxtbW(unsigned reg) { + // Sign-extend the low 8 bits in-place for int8 arguments/returns. + return 0x13001c00 | (reg << 5) | reg; +} + +uint32_t UxtbW(unsigned reg) { + // Zero-extend the low 8 bits in-place for bool/uint8 arguments/returns. + return 0x53001c00 | (reg << 5) | reg; +} + +uint32_t SxthW(unsigned reg) { + // Sign-extend the low 16 bits in-place for int16 arguments/returns. + return 0x13003c00 | (reg << 5) | reg; +} + +uint32_t UxthW(unsigned reg) { + // Zero-extend the low 16 bits in-place for uint16 arguments/returns. + return 0x53003c00 | (reg << 5) | reg; +} + +// Narrow integer parameters arrive widened by V8. Sign/zero extend them back +// to the ABI width expected by the native target before the final call. +bool EmitNarrow(uint32_t** cursor, FastFFIType type, unsigned reg) { + switch (type) { + case FastFFIType::kBool: + case FastFFIType::kUint8: + *(*cursor)++ = UxtbW(reg); + return true; + case FastFFIType::kInt8: + *(*cursor)++ = SxtbW(reg); + return true; + case FastFFIType::kUint16: + *(*cursor)++ = UxthW(reg); + return true; + case FastFFIType::kInt16: + *(*cursor)++ = SxthW(reg); + return true; + default: + return false; + } +} + +// Materialize an absolute address into x16, the scratch register used for BLR. +void EmitLoadX16(uint32_t** cursor, uintptr_t address) { + *(*cursor)++ = MovzX16((address >> 0) & 0xffff, 0); + *(*cursor)++ = MovkX16((address >> 16) & 0xffff, 16); + *(*cursor)++ = MovkX16((address >> 32) & 0xffff, 32); + *(*cursor)++ = MovkX16((address >> 48) & 0xffff, 48); +} + +// Stack adjustment must preserve the platform's 16-byte SP alignment rule. +unsigned Align16(unsigned value) { + return (value + 15) & ~15; +} + +} // namespace + +extern "C" bool node_ffi_create_fast_trampoline( + void* target, + const node::ffi::FastFFIType* args, + size_t argc, + node::ffi::FastFFIType result, + node::ffi::FastFFITrampoline* out) { + // Null inputs mean the caller cannot safely create executable code for this + // signature. Report rejection so the generic FFI path can be used instead. + if (target == nullptr || out == nullptr) { + return false; + } + + size_t gp_count = 0; + size_t fp_count = 0; + bool has_buffer_args = false; + // Count the incoming register classes before emitting code. This keeps all + // unsupported cases out of the code generator instead of relying on partial + // instruction streams to fail later. + for (size_t i = 0; i < argc; i++) { + if (IsFloatType(args[i])) { + fp_count++; + } else { + gp_count++; + has_buffer_args = has_buffer_args || IsPointerValueType(args[i]); + } + } + + // x0 is occupied by the V8 receiver, leaving x1..x7 for incoming GP args. + // Keep the first implementation register-only rather than generating stack + // shuffles incorrectly. + // Buffer conversion calls into C++ before invoking the target. Keep the + // first implementation scalar-only so no FP argument registers need to be + // preserved across helper calls. + if (has_buffer_args && fp_count != 0) { + return false; + } + + const size_t incoming_gp_count = gp_count + (has_buffer_args ? 1 : 0); + // When V8 passes FastApiCallbackOptions, it consumes one additional GP + // register after the declared user arguments. This implementation refuses + // signatures that would require loading incoming GP args from the stack. + if (incoming_gp_count > 7 || fp_count > 8) { + return false; + } + + constexpr size_t kMaxInstructions = 160; + const size_t code_size = kMaxInstructions * sizeof(uint32_t); + // The fixed allocation is intentionally larger than any currently emitted + // path. It keeps the generator simple while the supported signature set is + // small and bounded. + // Generate into writable anonymous memory first; the page is made executable + // only after the instruction stream is complete and the instruction cache is + // synchronized. + void* code = mmap(nullptr, + code_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, + -1, + 0); + if (code == MAP_FAILED) { + return false; + } + + uint32_t* cursor = static_cast(code); + // Save frame pointer and link register so helper calls and the final target + // call can return through this generated trampoline safely. + *cursor++ = kStpFpLrPreIndex; + + if (has_buffer_args) { + // Buffer conversion calls a C++ helper before the target call, so spill all + // incoming GP registers that may be clobbered by that helper. + const unsigned frame_size = Align16(incoming_gp_count * 8); + if (frame_size != 0) { + *cursor++ = SubSp(frame_size); + } + + for (unsigned i = 0; i < incoming_gp_count; i++) { + // V8 presents user GP args in x1.. and the callback options pointer after + // them. Spill that whole incoming register window at stack slot i. + *cursor++ = StrXSp(i + 1, i * 8); + } + + unsigned incoming_gp_index = 0; + unsigned target_gp_index = 0; + const unsigned options_offset = gp_count * 8; + // `incoming_gp_index` walks spilled V8 argument slots; `target_gp_index` + // walks the ABI registers expected by the native target. They differ only + // because V8 has an implicit receiver in x0 that the target does not see. + // Reload each original argument in target ABI order. Buffer arguments are + // converted to raw pointers; scalar GP arguments are moved/narrowed. + for (size_t i = 0; i < argc; i++) { + if (!IsGPType(args[i])) { + continue; + } + + const unsigned arg_offset = incoming_gp_index * 8; + if (IsPointerValueType(args[i])) { + // Call node_ffi_fast_buffer_data(value, options, index). A sentinel + // return means the helper already threw, so return nullptr/zero without + // invoking the native target. + *cursor++ = LdrXSp(0, arg_offset); + *cursor++ = LdrXSp(1, options_offset); + *cursor++ = MovzW(2, static_cast(i)); + EmitLoadX16(&cursor, + reinterpret_cast(node_ffi_fast_buffer_data)); + // blr x16 clobbers caller-saved registers. The original arguments were + // spilled above specifically so subsequent target args can be restored. + *cursor++ = kBlrX16; + EmitLoadX16(&cursor, std::numeric_limits::max()); + *cursor++ = CmpX(0, 16); + // If x0 != sentinel, skip over the early-return sequence and continue + // assembling the target call state. The offset differs by one + // instruction depending on whether stack cleanup is needed. + *cursor++ = BCond(frame_size != 0 ? 5 : 4, 1); // b.ne valid_buffer + // Conversion failed. Return zero/null directly; the helper has already + // scheduled the JS exception through FastApiCallbackOptions. + *cursor++ = MovX(0, 31); + if (frame_size != 0) { + *cursor++ = AddSp(frame_size); + } + *cursor++ = kLdpFpLrPostIndex; + *cursor++ = kRet; + if (target_gp_index != 0) { + // Helper returns the converted pointer in x0; move it into the ABI + // register assigned to this target argument if needed. + *cursor++ = MovX(target_gp_index, 0); + } + } else { + // Non-buffer GP arguments can be restored directly from the spill area. + *cursor++ = LdrXSp(target_gp_index, arg_offset); + EmitNarrow(&cursor, args[i], target_gp_index); + } + + incoming_gp_index++; + target_gp_index++; + } + } else { + // Scalar-only signatures need no spills: V8's receiver occupies x0, so + // user GP arguments are shifted down from x1..x7 to x0..x6. + unsigned gp_index = 0; + for (size_t i = 0; i < argc; i++) { + if (!IsGPType(args[i])) { + // FP arguments are already in v0..v7 and are not shifted by the x0 + // receiver slot, so the scalar path does not need to touch them. + continue; + } + *cursor++ = MovX(gp_index, gp_index + 1); + EmitNarrow(&cursor, args[i], gp_index); + gp_index++; + } + } + + // Tail of the trampoline: load the actual library symbol address and call it + // with arguments now arranged according to the native ABI. + EmitLoadX16(&cursor, reinterpret_cast(target)); + *cursor++ = kBlrX16; + + if (has_buffer_args) { + // Restore the spill frame allocated for buffer conversion before returning + // through the saved frame pointer/link-register pair. + const unsigned frame_size = Align16(incoming_gp_count * 8); + if (frame_size != 0) { + *cursor++ = AddSp(frame_size); + } + } + + *cursor++ = kLdpFpLrPostIndex; + // Native small integer returns may leave upper bits unspecified. Normalize + // them before V8 observes the result. + EmitNarrow(&cursor, result, 0); + *cursor++ = kRet; + + const size_t written = reinterpret_cast(cursor) - + static_cast(code); + +#if defined(__APPLE__) + // Make the just-written instructions visible to the CPU's instruction cache. + sys_icache_invalidate(code, written); +#else + __builtin___clear_cache(static_cast(code), + static_cast(code) + written); +#endif + + // Enforce W^X after code generation: the trampoline is executable but no + // longer writable once published through FastFFITrampoline. + if (mprotect(code, code_size, PROT_READ | PROT_EXEC) != 0) { + munmap(code, code_size); + return false; + } + + out->code = code; + out->size = code_size; + return true; +} + +extern "C" void node_ffi_free_fast_trampoline( + node::ffi::FastFFITrampoline* trampoline) { + // Trampoline cleanup is idempotent so metadata destruction can safely run on + // partially initialized or already-released objects. + if (trampoline == nullptr || trampoline->code == nullptr) { + return; + } + munmap(trampoline->code, trampoline->size); + trampoline->code = nullptr; + trampoline->size = 0; +} + +#else + +extern "C" bool node_ffi_create_fast_trampoline( + void* target, + const node::ffi::FastFFIType* args, + size_t argc, + node::ffi::FastFFIType result, + node::ffi::FastFFITrampoline* out) { + // Non-AArch64 platforms have no generated trampoline yet. Returning false + // lets CreateFastFFIMetadata() fall back cleanly. + return false; +} + +extern "C" void node_ffi_free_fast_trampoline( + node::ffi::FastFFITrampoline* trampoline) { + // No code is allocated in the non-AArch64 stub. +} + +#endif // defined(__aarch64__) || defined(_M_ARM64) + +#endif // HAVE_FFI diff --git a/src/node_ffi.cc b/src/node_ffi.cc index 22aeb8fd22f108..0371d79b638917 100644 --- a/src/node_ffi.cc +++ b/src/node_ffi.cc @@ -8,6 +8,7 @@ #include "base_object-inl.h" #include "env-inl.h" #include "ffi/data.h" +#include "ffi/fast.h" #include "ffi/types.h" #include "node_errors.h" @@ -212,17 +213,49 @@ MaybeLocal DynamicLibrary::CreateFunction( DCHECK_EQ(fn->args.size(), fn->arg_type_names.size()); - bool use_sb = IsSBEligibleSignature(*fn); + // Try the generated Fast API path first. If metadata creation rejects the + // signature, fall back to SharedBuffer for supported scalar shapes, then to + // the generic libffi invoker. + info->fast_metadata = CreateFastFFIMetadata(*fn); + bool use_fast_api = info->fast_metadata != nullptr; + bool use_sb = !use_fast_api && IsSBEligibleSignature(*fn); bool has_ptr_args = use_sb && SignatureHasPointerArgs(*fn); + // Fast API signatures that still accept JS pointer-like values need a JS + // wrapper with the native type names attached as hidden metadata. + bool needs_raw_pointer_conversions = + use_fast_api && SignatureNeedsRawPointerConversions(*fn); + // A single pointer-like parameter can get a separate Buffer-aware Fast API + // entrypoint so Buffer calls avoid JS pointer extraction. + bool needs_fast_buffer_invoke = + use_fast_api && SignatureNeedsFastBufferInvoke(*fn); Local data = External::New(isolate, info.get(), v8::kExternalPointerTypeTagDefault); - MaybeLocal maybe_ret = - Function::New(context, - use_sb ? DynamicLibrary::InvokeFunctionSB - : DynamicLibrary::InvokeFunction, - data); + MaybeLocal maybe_ret; + if (use_fast_api) { + // V8 calls this FunctionTemplate through `fast_metadata->c_function` when + // the optimized Fast API call path is available. The normal callback stays + // attached as a fallback for V8 deopts and unsupported call sites. + Local tmpl = FunctionTemplate::New( + isolate, + DynamicLibrary::InvokeFunction, + data, + Local(), + fn->args.size(), + v8::ConstructorBehavior::kThrow, + v8::SideEffectType::kHasSideEffect, + &info->fast_metadata->c_function); + maybe_ret = tmpl->GetFunction(context); + } else { + // Non-Fast signatures either use the SharedBuffer invoker, where JS writes + // argument slots before calling with no arguments, or the generic invoker + // that converts each JS argument in C++. + maybe_ret = Function::New(context, + use_sb ? DynamicLibrary::InvokeFunctionSB + : DynamicLibrary::InvokeFunction, + data); + } Local ret; if (!maybe_ret.ToLocal(&ret)) { return MaybeLocal(); @@ -252,6 +285,9 @@ MaybeLocal DynamicLibrary::CreateFunction( static_cast(ReadOnly | DontEnum | DontDelete); if (use_sb) { + // SharedBuffer layout is intentionally fixed-width: slot 0 stores the + // return value and slots 1..N store argument payloads. The JS wrapper and + // InvokeFunctionSB share this exact layout. size_t sb_size = 8 * (fn->args.size() + 1); Local ab = ArrayBuffer::New(isolate, sb_size); // The shared_ptr to the backing store keeps the memory alive while @@ -313,6 +349,60 @@ MaybeLocal DynamicLibrary::CreateFunction( } } + if (needs_raw_pointer_conversions || needs_fast_buffer_invoke) { + // Fast API wrappers need only the parameter type names. Result conversion + // is still handled by V8's CFunction metadata, unlike the SharedBuffer path + // which must also know how to read slot 0. + Local params_arr; + if (!ToV8Value(context, fn->arg_type_names, isolate).ToLocal(¶ms_arr)) { + return MaybeLocal(); + } + if (!ret->DefineOwnProperty(context, + env->ffi_fast_params_symbol(), + params_arr, + internal_attrs) + .FromMaybe(false)) { + return MaybeLocal(); + } + } + + if (needs_fast_buffer_invoke) { + // Build an alternate CFunction that describes the pointer-like argument as + // a V8 buffer value. The JS wrapper dispatches here only when the runtime + // argument is Buffer/ArrayBuffer-backed memory. + std::shared_ptr fast_buffer_fn = + CloneWithFastBufferArgNames(fn); + info->fast_buffer_metadata = CreateFastFFIMetadata(*fast_buffer_fn); + if (info->fast_buffer_metadata != nullptr) { + // Store the secondary invoker on the primary raw function under a hidden + // Symbol. Keeping it separate avoids overloading SharedBuffer slow-path + // metadata for Fast API routing. + Local tmpl = FunctionTemplate::New( + isolate, + DynamicLibrary::InvokeFunction, + data, + Local(), + fn->args.size(), + v8::ConstructorBehavior::kThrow, + v8::SideEffectType::kHasSideEffect, + &info->fast_buffer_metadata->c_function); + Local fast_buffer_invoke; + if (!tmpl->GetFunction(context).ToLocal(&fast_buffer_invoke)) { + return MaybeLocal(); + } + if (!ret->DefineOwnProperty(context, + env->ffi_fast_buffer_invoke_symbol(), + fast_buffer_invoke, + internal_attrs) + .FromMaybe(false)) { + return MaybeLocal(); + } + } + } + + // The generated JS function owns FFIFunctionInfo through a weak callback. + // The persistent library reference keeps the dlopen handle alive while any + // generated function can still call into it. info->self.Reset(isolate, ret); info->library.Reset(isolate, object()); info->self.SetWeak(info.release(), @@ -1171,6 +1261,8 @@ static void Initialize(Local target, FIXED_ONE_BYTE_STRING(isolate, "kSbSharedBuffer"), env->ffi_sb_shared_buffer_symbol()) .Check(); + // Fast API wrappers use separate metadata Symbols so pointer-conversion + // routing does not depend on SharedBuffer internals. target ->Set(context, FIXED_ONE_BYTE_STRING(isolate, "kSbInvokeSlow"), @@ -1186,6 +1278,16 @@ static void Initialize(Local target, FIXED_ONE_BYTE_STRING(isolate, "kSbResult"), env->ffi_sb_result_symbol()) .Check(); + target + ->Set(context, + FIXED_ONE_BYTE_STRING(isolate, "kFastParams"), + env->ffi_fast_params_symbol()) + .Check(); + target + ->Set(context, + FIXED_ONE_BYTE_STRING(isolate, "kFastBufferInvoke"), + env->ffi_fast_buffer_invoke_symbol()) + .Check(); } } // namespace ffi diff --git a/src/node_ffi.h b/src/node_ffi.h index 22d78a2ad4f35a..bf5a67b5c9fc5e 100644 --- a/src/node_ffi.h +++ b/src/node_ffi.h @@ -3,6 +3,7 @@ #if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS #include "base_object.h" +#include "ffi/fast.h" #include "ffi.h" #include "uv.h" @@ -35,6 +36,8 @@ struct FFIFunctionInfo { std::shared_ptr sb_backing; // Keep the owning DynamicLibrary alive while the generated function is alive. v8::Global library; + std::unique_ptr fast_metadata; + std::unique_ptr fast_buffer_metadata; }; struct FFICallback { diff --git a/test/ffi/fixture_library/ffi_test_library.c b/test/ffi/fixture_library/ffi_test_library.c index 4a57b9c9970a9a..8eb17da3abb717 100644 --- a/test/ffi/fixture_library/ffi_test_library.c +++ b/test/ffi/fixture_library/ffi_test_library.c @@ -11,6 +11,9 @@ // Integer operations. +FFI_EXPORT void noop_void(void) { +} + FFI_EXPORT int8_t add_i8(int8_t a, int8_t b) { return a + b; } @@ -43,6 +46,10 @@ FFI_EXPORT uint64_t add_u64(uint64_t a, uint64_t b) { return a + b; } +FFI_EXPORT int32_t identity_i32(int32_t value) { + return value; +} + FFI_EXPORT char identity_char(char value) { return value; } @@ -75,6 +82,11 @@ FFI_EXPORT void* identity_pointer(void* ptr) { return ptr; } +FFI_EXPORT uint8_t is_null_pointer(void* ptr) { + // NOLINTNEXTLINE (readability/null_usage) + return ptr == NULL; +} + FFI_EXPORT uint64_t pointer_to_usize(void* ptr) { return (uint64_t)(uintptr_t)ptr; } @@ -89,6 +101,14 @@ FFI_EXPORT uint64_t string_length(const char* str) { return str ? strlen(str) : 0; } +FFI_EXPORT uint8_t string_first_char(const char* str) { + return str ? (uint8_t)str[0] : 0; +} + +FFI_EXPORT uint8_t string_equals_hello(const char* str) { + return str && strcmp(str, "hello") == 0; +} + FFI_EXPORT char* string_concat(const char* a, const char* b) { if (!a || !b) { // NOLINTNEXTLINE (readability/null_usage) @@ -133,6 +153,10 @@ FFI_EXPORT void free_string(char* str) { // Buffer/Array operations. +FFI_EXPORT uint8_t first_byte(const uint8_t* buffer) { + return buffer ? buffer[0] : 0; +} + FFI_EXPORT void fill_buffer(uint8_t* buffer, uint64_t length, uint32_t value) { if (!buffer) { return; @@ -368,6 +392,11 @@ sum_five_i32(int32_t a, int32_t b, int32_t c, int32_t d, int32_t e) { return a + b + c + d + e; } +FFI_EXPORT int32_t +sum_5_i32(int32_t a, int32_t b, int32_t c, int32_t d, int32_t e) { + return a + b + c + d + e; +} + FFI_EXPORT double sum_five_f64( double a, double b, double c, double d, double e) { return a + b + c + d + e; @@ -386,6 +415,17 @@ sum_6_i32(int32_t a, int32_t b, int32_t c, int32_t d, int32_t e, int32_t f) { return a + b + c + d + e + f; } +FFI_EXPORT int32_t sum_8_i32(int32_t a, + int32_t b, + int32_t c, + int32_t d, + int32_t e, + int32_t f, + int32_t g, + int32_t h) { + return a + b + c + d + e + f + g + h; +} + FFI_EXPORT int32_t sum_7_i32(int32_t a, int32_t b, int32_t c, diff --git a/test/ffi/test-ffi-calls.js b/test/ffi/test-ffi-calls.js index ef43fb0a6f7274..acec70e33bede3 100644 --- a/test/ffi/test-ffi-calls.js +++ b/test/ffi/test-ffi-calls.js @@ -112,6 +112,47 @@ test('ffi strings and buffers cross the boundary correctly', () => { } }); +test('ffi string signatures convert strings to temporary pointers', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + string_length: { parameters: ['string'], result: 'u64' }, + safe_strlen: { parameters: ['str'], result: 'i32' }, + }); + try { + assert.strictEqual(functions.string_length('hello ffi'), 9n); + assert.strictEqual(functions.safe_strlen('hello ffi'), 9); + assert.strictEqual(functions.safe_strlen(null), -1); + assert.strictEqual(functions.safe_strlen(undefined), -1); + } finally { + lib.close(); + } +}); + +test('ffi buffer and ArrayBuffer signatures pass backing-store pointers', () => { + { + const { lib, functions } = ffi.dlopen(libraryPath, { + first_byte: { parameters: ['buffer'], result: 'u8' }, + }); + try { + assert.strictEqual(functions.first_byte(Buffer.from([42, 1])), 42); + assert.strictEqual(functions.first_byte(new Uint8Array([43, 1])), 43); + } finally { + lib.close(); + } + } + + { + const { lib, functions } = ffi.dlopen(libraryPath, { + first_byte: { parameters: ['arraybuffer'], result: 'u8' }, + }); + try { + const ab = new Uint8Array([44, 1]).buffer; + assert.strictEqual(functions.first_byte(ab), 44); + } finally { + lib.close(); + } + } +}); + test('ffi typed array accessors work', () => { const { lib, functions: symbols } = getLibrary(); try { diff --git a/test/ffi/test-ffi-fast-buffer.js b/test/ffi/test-ffi-fast-buffer.js new file mode 100644 index 00000000000000..fe2f47b0f9db48 --- /dev/null +++ b/test/ffi/test-ffi-fast-buffer.js @@ -0,0 +1,71 @@ +// Flags: --experimental-ffi --expose-internals +'use strict'; + +const common = require('../common'); +common.skipIfFFIMissing(); + +const assert = require('node:assert'); +const { test } = require('node:test'); +const { internalBinding } = require('internal/test/binding'); +const { + kSbSharedBuffer, +} = internalBinding('ffi'); + +const ffi = require('node:ffi'); +const { libraryPath } = require('./ffi-test-common'); + +test('fast FFI accepts buffer and arraybuffer arguments natively', () => { + const lib = new ffi.DynamicLibrary(libraryPath); + const functions = { + first_byte_buffer: lib.getFunction('first_byte', { + parameters: ['buffer'], + result: 'u8', + }), + first_byte_arraybuffer: lib.getFunction('first_byte', { + parameters: ['arraybuffer'], + result: 'u8', + }), + pointer_to_usize: lib.getFunction('pointer_to_usize', { + parameters: ['pointer'], + result: 'u64', + }), + sum_buffer: { + parameters: ['buffer', 'u64'], + result: 'u64', + }, + }; + functions.sum_buffer = lib.getFunction('sum_buffer', functions.sum_buffer); + + try { + const bytes = Buffer.from([1, 2, 3, 4]); + const ab = Uint8Array.from([5, 6, 7, 8]).buffer; + + assert.strictEqual(functions.first_byte_buffer(bytes), 1); + assert.strictEqual(functions.first_byte_arraybuffer(ab), 5); + assert.strictEqual( + functions.pointer_to_usize(bytes), ffi.getRawPointer(bytes)); + assert.strictEqual(functions.sum_buffer(bytes, BigInt(bytes.length)), 10n); + + if (process.arch === 'arm64') { + assert.strictEqual(functions.first_byte_buffer[kSbSharedBuffer], undefined); + assert.strictEqual(functions.sum_buffer[kSbSharedBuffer], undefined); + assert.strictEqual(functions.pointer_to_usize[kSbSharedBuffer], undefined); + } + } finally { + lib.close(); + } +}); + +test('fast FFI buffer arguments reject invalid values', () => { + const { lib, functions } = ffi.dlopen(libraryPath, { + first_byte: { parameters: ['buffer'], result: 'u8' }, + }); + + try { + assert.throws(() => functions.first_byte(123), { + code: 'ERR_INVALID_ARG_VALUE', + }); + } finally { + lib.close(); + } +}); diff --git a/test/ffi/test-ffi-shared-buffer.js b/test/ffi/test-ffi-shared-buffer.js index 43d76a4da18315..33ddc6218bc1c3 100644 --- a/test/ffi/test-ffi-shared-buffer.js +++ b/test/ffi/test-ffi-shared-buffer.js @@ -13,6 +13,7 @@ const { test } = require('node:test'); const { internalBinding } = require('internal/test/binding'); const ffiBinding = internalBinding('ffi'); const { + kFastParams, kSbInvokeSlow, kSbParams, kSbResult, @@ -130,14 +131,16 @@ test('pointer args: fast path (BigInt/null) and slow-path fallback (Buffer/Array } }); -test('string pointer uses slow-path fallback', () => { +test('string pointer uses shared-buffer pointer conversion', () => { const { lib, functions } = ffi.dlopen(libraryPath, { string_length: { result: 'u64', parameters: ['pointer'] }, + safe_strlen: { result: 'i32', parameters: ['string'] }, }); try { assert.strictEqual(functions.string_length('hello'), 5n); - // strlen(NULL) is UB, so use a NUL-terminated Buffer for the fast path. assert.strictEqual(functions.string_length(Buffer.from('world\0')), 5n); + assert.strictEqual(functions.safe_strlen('hello'), 5); + assert.strictEqual(functions.safe_strlen(null), -1); } finally { lib.close(); } @@ -356,6 +359,7 @@ test('SB metadata is Symbol-keyed, attribute-hardened, and not leaked onto the w rawLib, 'add_i32', { result: 'i32', parameters: ['i32', 'i32'] }); for (const [name, sym] of [ + ['kFastParams', kFastParams], ['kSbSharedBuffer', kSbSharedBuffer], ['kSbInvokeSlow', kSbInvokeSlow], ['kSbParams', kSbParams], @@ -364,35 +368,41 @@ test('SB metadata is Symbol-keyed, attribute-hardened, and not leaked onto the w assert.strictEqual(typeof sym, 'symbol', `${name} must be a Symbol`); } - // Numeric-only signature: kSbInvokeSlow absent; the rest present and hardened. + // Fast-API-eligible signatures bypass the SB wrapper and therefore do not + // carry SB metadata. for (const [name, sym] of [ ['kSbSharedBuffer', kSbSharedBuffer], ['kSbParams', kSbParams], ['kSbResult', kSbResult], + ['kSbInvokeSlow', kSbInvokeSlow], ]) { const desc = Object.getOwnPropertyDescriptor(rawFn, sym); - assert.ok(desc !== undefined, `${name} missing on pure-numeric SB function`); - assert.strictEqual(desc.enumerable, false); - assert.strictEqual(desc.configurable, false); - assert.strictEqual(desc.writable, false); + assert.strictEqual(desc, undefined, `${name} present on Fast API function`); } - assert.strictEqual( - Object.getOwnPropertyDescriptor(rawFn, kSbInvokeSlow), undefined); - // Pointer signature: kSbInvokeSlow must exist (and be hardened). - const rawPtrFn = rawGetFunctionUnpatched.call( - rawLib, 'identity_pointer', { result: 'pointer', parameters: ['pointer'] }); - const slowDesc = Object.getOwnPropertyDescriptor(rawPtrFn, kSbInvokeSlow); - assert.ok(slowDesc !== undefined); - assert.strictEqual(slowDesc.enumerable, false); - assert.strictEqual(slowDesc.configurable, false); - assert.strictEqual(slowDesc.writable, false); + // Fast string signatures carry parameter metadata so the JS wrapper can + // perform string-to-pointer conversion, but still do not carry SB state. + const rawStringFn = rawGetFunctionUnpatched.call( + rawLib, 'safe_strlen', { result: 'u64', parameters: ['string'] }); + const paramsDesc = Object.getOwnPropertyDescriptor(rawStringFn, kFastParams); + assert.ok(paramsDesc !== undefined, 'kFastParams missing on Fast string function'); + assert.strictEqual(paramsDesc.enumerable, false); + assert.strictEqual(paramsDesc.configurable, false); + assert.strictEqual(paramsDesc.writable, false); + assert.strictEqual( + Object.getOwnPropertyDescriptor(rawStringFn, kSbParams), undefined); + assert.strictEqual( + Object.getOwnPropertyDescriptor(rawStringFn, kSbSharedBuffer), undefined); + assert.strictEqual( + Object.getOwnPropertyDescriptor(rawStringFn, kSbResult), undefined); + assert.strictEqual( + Object.getOwnPropertyDescriptor(rawStringFn, kSbInvokeSlow), undefined); assert.deepStrictEqual(Object.keys(rawFn), ['pointer']); const ownSyms = Object.getOwnPropertySymbols(rawFn); - assert.ok(ownSyms.includes(kSbSharedBuffer)); - assert.ok(ownSyms.includes(kSbParams)); - assert.ok(ownSyms.includes(kSbResult)); + assert.ok(!ownSyms.includes(kSbSharedBuffer)); + assert.ok(!ownSyms.includes(kSbParams)); + assert.ok(!ownSyms.includes(kSbResult)); // Internals must not be forwarded by `inheritMetadata`. const { lib, functions } = ffi.dlopen(libraryPath, {